@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 996357e480
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Separate out profiling-related logic (#11746)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 6, 2025 at 16:02 UTC 996357e4808ca5eab97d4c97c7d25b3073f46aab
17 files changed +1015 -718
tests/multimodal/test_processing.py
+4 -3
@@ -586,9 +586,10 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
586 )
587
588 processor = processor_factory(ctx, cache=None)
589 + profiler = processor.profiling_info
590
591 mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
591 - processor.get_supported_mm_limits = mock_supported_mm_limits
592 + profiler.get_supported_mm_limits = mock_supported_mm_limits
593
594 if is_valid:
595 exc_ctx = nullcontext()
@@ -596,7 +597,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
597 exc_ctx = pytest.raises(ValueError, match="this model only supports")
598
599 with exc_ctx:
599 - processor._get_and_validate_dummy_mm_counts()
600 + profiler.get_mm_limits()
601
602
603 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@@ -723,7 +724,7 @@ def _test_processing_cache_correctness(
724 }
725
726 mm_counts = {k: len(vs) for k, vs in mm_data.items()}
726 - prompt = baseline_processor._get_dummy_processor_inputs(
727 + prompt = baseline_processor.profiling_info.get_dummy_processor_inputs(
728 model_config.max_model_len,
729 mm_counts,
730 ).prompt_text
vllm/model_executor/models/aria.py
+47 -32
@@ -24,8 +24,9 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
24 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
25 NestedTensors)
26 from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 - MultiModalDataItems, ProcessorInputs,
27 + MultiModalDataItems, ProcessingMixin,
28 PromptReplacement)
29 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
30 from vllm.sequence import IntermediateTensors
31 from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
32 AriaVisionConfig)
@@ -444,18 +445,58 @@ def build_mm_projector(config: PretrainedConfig):
445 )
446
447
447 -class AriaMultiModalProcessor(BaseMultiModalProcessor):
448 +class AriaProcessingMixin(ProcessingMixin):
449
449 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
450 - return {"image": None}
450 + def _get_hf_config(self):
451 + return self.ctx.get_hf_config()
452 +
453 + def _get_vision_config(self) -> AriaVisionConfig:
454 + return self._get_hf_config().vision_config
455
456 def _get_num_image_tokens(self) -> int:
453 - hf_config = self.ctx.get_hf_config()
457 + hf_config = self._get_hf_config()
458 return max(hf_config.projector_patch_to_query_dict.values())
459
460 +
461 +class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
462 +
463 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
464 + return {"image": None}
465 +
466 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
467 return {"image": self._get_num_image_tokens()}
468
469 + def get_dummy_processor_inputs(
470 + self,
471 + seq_len: int,
472 + mm_counts: Mapping[str, int],
473 + ) -> ProcessorInputs:
474 + vision_config = self._get_vision_config()
475 +
476 + max_image_size = vision_config.image_size
477 + num_images = mm_counts.get("image", 0)
478 +
479 + mm_data = {
480 + "image":
481 + self._get_dummy_images(width=max_image_size,
482 + height=max_image_size,
483 + num_images=num_images)
484 + }
485 +
486 + hf_processor = self._get_hf_processor()
487 + image_token: str = hf_processor.image_token # type: ignore
488 +
489 + return ProcessorInputs(
490 + prompt_text=image_token * num_images,
491 + mm_data=mm_data,
492 + )
493 +
494 +
495 +class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
496 +
497 + def _get_profiling_info(self) -> BaseProfilingInfo:
498 + return AriaProfilingInfo(self.ctx)
499 +
500 def _get_mm_fields_config(
501 self,
502 hf_inputs: BatchFeature,
@@ -472,7 +513,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
513 hf_processor_mm_kwargs: Mapping[str, object],
514 out_mm_kwargs: MultiModalKwargs,
515 ) -> list[PromptReplacement]:
475 - hf_config = self.ctx.get_hf_config()
516 + hf_config = self._get_hf_config()
517 image_token_id = hf_config.image_token_index
518
519 num_image_tokens = self._get_num_image_tokens()
@@ -485,32 +526,6 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
526 )
527 ]
528
488 - def _get_dummy_processor_inputs(
489 - self,
490 - seq_len: int,
491 - mm_counts: Mapping[str, int],
492 - ) -> ProcessorInputs:
493 - hf_config = self.ctx.get_hf_config()
494 - vision_config: AriaVisionConfig = hf_config.vision_config
495 -
496 - max_image_size = vision_config.image_size
497 - num_images = mm_counts.get("image", 0)
498 -
499 - mm_data = {
500 - "image":
501 - self._get_dummy_images(width=max_image_size,
502 - height=max_image_size,
503 - num_images=num_images)
504 - }
505 -
506 - hf_processor = self._get_hf_processor()
507 - image_token: str = hf_processor.image_token # type: ignore
508 -
509 - return ProcessorInputs(
510 - prompt_text=image_token * num_images,
511 - mm_data=mm_data,
512 - )
513 -
529
530 @MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
531 class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
vllm/model_executor/models/blip2.py
+44 -34
@@ -4,8 +4,8 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
5 import torch
6 import torch.nn as nn
7 -from transformers import (BatchFeature, Blip2Config, Blip2Processor,
8 - Blip2QFormerConfig, apply_chunking_to_forward)
7 +from transformers import (BatchFeature, Blip2Config, Blip2QFormerConfig,
8 + apply_chunking_to_forward)
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, VllmConfig
@@ -18,8 +18,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18 MultiModalInputsV2, MultiModalKwargs,
19 NestedTensors, PlaceholderRange)
20 from vllm.multimodal.processing import (BaseMultiModalProcessor,
21 - MultiModalDataItems, ProcessorInputs,
21 + MultiModalDataItems, ProcessingMixin,
22 PromptReplacement)
23 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
24 from vllm.sequence import IntermediateTensors
25
26 from .blip import BlipVisionModel
@@ -396,20 +397,52 @@ class Blip2QFormerModel(nn.Module):
397 return sequence_output
398
399
399 -class Blip2MultiModalProcessor(BaseMultiModalProcessor):
400 +class Blip2ProcessingMixin(ProcessingMixin):
401
401 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
402 - return {"image": 1}
402 + def _get_hf_config(self):
403 + return self.ctx.get_hf_config(Blip2Config)
404
405 def _get_num_image_tokens(self) -> int:
405 - hf_config = self.ctx.get_hf_config(Blip2Config)
406 + hf_config = self._get_hf_config()
407 return hf_config.num_query_tokens
408
409 +
410 +class Blip2ProfilingInfo(Blip2ProcessingMixin, BaseProfilingInfo):
411 +
412 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
413 + return {"image": 1}
414 +
415 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
416 return {"image": self._get_num_image_tokens()}
417
411 - def _get_hf_processor(self) -> Blip2Processor:
412 - return self.ctx.get_hf_processor(Blip2Processor)
418 + def get_dummy_processor_inputs(
419 + self,
420 + seq_len: int,
421 + mm_counts: Mapping[str, int],
422 + ) -> ProcessorInputs:
423 + hf_config = self._get_hf_config()
424 + vision_config = hf_config.vision_config
425 +
426 + max_image_size = vision_config.image_size
427 + num_images = mm_counts.get("image", 0)
428 +
429 + mm_data = {
430 + "image":
431 + self._get_dummy_images(width=max_image_size,
432 + height=max_image_size,
433 + num_images=num_images)
434 + }
435 +
436 + return ProcessorInputs(
437 + prompt_text="",
438 + mm_data=mm_data,
439 + )
440 +
441 +
442 +class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
443 +
444 + def _get_profiling_info(self) -> BaseProfilingInfo:
445 + return Blip2ProfilingInfo(self.ctx)
446
447 def _get_mm_fields_config(
448 self,
@@ -427,13 +460,13 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
460 hf_processor_mm_kwargs: Mapping[str, object],
461 out_mm_kwargs: MultiModalKwargs,
462 ) -> list[PromptReplacement]:
430 - max_image_tokens = self._get_num_image_tokens()
463 + num_image_tokens = self._get_num_image_tokens()
464
465 return [
466 PromptReplacement(
467 modality="image",
468 target="</s>",
436 - replacement="<image>" * max_image_tokens + "</s>",
469 + replacement="<image>" * num_image_tokens + "</s>",
470 )
471 ]
472
@@ -457,29 +490,6 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
490
491 return result
492
460 - def _get_dummy_processor_inputs(
461 - self,
462 - seq_len: int,
463 - mm_counts: Mapping[str, int],
464 - ) -> ProcessorInputs:
465 - hf_config = self.ctx.get_hf_config(Blip2Config)
466 - vision_config = hf_config.vision_config
467 -
468 - max_image_size = vision_config.image_size
469 - num_images = mm_counts.get("image", 0)
470 -
471 - mm_data = {
472 - "image":
473 - self._get_dummy_images(width=max_image_size,
474 - height=max_image_size,
475 - num_images=num_images)
476 - }
477 -
478 - return ProcessorInputs(
479 - prompt_text="",
480 - mm_data=mm_data,
481 - )
482 -
493
494 @MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
495 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
vllm/model_executor/models/chameleon.py
+43 -29
@@ -31,8 +31,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31 MultiModalInputsV2, MultiModalKwargs,
32 NestedTensors, PlaceholderRange)
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 - MultiModalDataItems, ProcessorInputs,
34 + MultiModalDataItems, ProcessingMixin,
35 PromptReplacement)
36 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
37 from vllm.sequence import IntermediateTensors
38 from vllm.utils import print_warning_once
39
@@ -48,20 +49,55 @@ class ChameleonImagePixelInputs(TypedDict):
49 """Shape: `(batch_size * num_images, num_channels, height, width)`"""
50
51
51 -class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
52 +class ChameleonProcessingMixin(ProcessingMixin):
53
53 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
54 - return {"image": 1}
54 + def _get_hf_config(self):
55 + return self.ctx.get_hf_config(ChameleonConfig)
56 +
57 + def _get_hf_processor(self):
58 + return self.ctx.get_hf_processor(ChameleonProcessor)
59
60 def _get_num_image_tokens(self) -> int:
61 processor = self._get_hf_processor()
62 return processor.image_seq_length
63
64 +
65 +class ChameleonProfilingInfo(ChameleonProcessingMixin, BaseProfilingInfo):
66 +
67 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
68 + return {"image": 1}
69 +
70 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
71 return {"image": self._get_num_image_tokens()}
72
63 - def _get_hf_processor(self) -> ChameleonProcessor:
64 - return self.ctx.get_hf_processor(ChameleonProcessor)
73 + def get_dummy_processor_inputs(
74 + self,
75 + seq_len: int,
76 + mm_counts: Mapping[str, int],
77 + ) -> ProcessorInputs:
78 + config = self._get_hf_config()
79 +
80 + width = height = config.vq_config.resolution
81 + num_images = mm_counts.get("image", 0)
82 +
83 + mm_data = {
84 + "image":
85 + self._get_dummy_images(width=width,
86 + height=height,
87 + num_images=num_images)
88 + }
89 +
90 + return ProcessorInputs(
91 + prompt_text="<image>" * num_images,
92 + mm_data=mm_data,
93 + )
94 +
95 +
96 +class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
97 + BaseMultiModalProcessor):
98 +
99 + def _get_profiling_info(self) -> BaseProfilingInfo:
100 + return ChameleonProfilingInfo(self.ctx)
101
102 def _get_mm_fields_config(
103 self,
@@ -76,7 +112,7 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
112 hf_processor_mm_kwargs: Mapping[str, object],
113 out_mm_kwargs: MultiModalKwargs,
114 ) -> list[PromptReplacement]:
79 - processor = self._get_hf_processor()
115 + processor = self._get_hf_processor(**hf_processor_mm_kwargs)
116
117 return [
118 PromptReplacement(
@@ -90,28 +126,6 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
126 )
127 ]
128
93 - def _get_dummy_processor_inputs(
94 - self,
95 - seq_len: int,
96 - mm_counts: Mapping[str, int],
97 - ) -> ProcessorInputs:
98 - config = self.ctx.get_hf_config(ChameleonConfig)
99 -
100 - width = height = config.vq_config.resolution
101 - num_images = mm_counts.get("image", 0)
102 -
103 - mm_data = {
104 - "image":
105 - self._get_dummy_images(width=width,
106 - height=height,
107 - num_images=num_images)
108 - }
109 -
110 - return ProcessorInputs(
111 - prompt_text="<image>" * num_images,
112 - mm_data=mm_data,
113 - )
114 -
129 def apply(
130 self,
131 prompt_text: str,
vllm/model_executor/models/fuyu.py
+50 -35
@@ -35,8 +35,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35 NestedTensors, PlaceholderRange)
36 from vllm.multimodal.parse import ImageProcessorItems, ImageSize
37 from vllm.multimodal.processing import (BaseMultiModalProcessor,
38 - MultiModalDataItems, ProcessorInputs,
38 + MultiModalDataItems, ProcessingMixin,
39 PromptReplacement)
40 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
41 from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsMultiModal, SupportsPP
@@ -63,18 +64,16 @@ class FuyuImagePatchInputs(TypedDict):
64 """
65
66
66 -class FuyuMultiModalProcessor(BaseMultiModalProcessor):
67 +class FuyuProcessingMixin(ProcessingMixin):
68
68 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
69 - return {"image": 1}
69 + def _get_hf_config(self):
70 + return self.ctx.get_hf_config(FuyuConfig)
71
71 - def _get_image_target_size(self) -> ImageSize:
72 - processor = self._get_hf_processor()
73 - image_processor: FuyuImageProcessor = processor.image_processor
72 + def _get_hf_processor(self):
73 + return self.ctx.get_hf_processor(FuyuProcessor)
74
75 - target_size = image_processor.size
76 - return ImageSize(width=target_size["width"],
77 - height=target_size["height"])
75 + def _get_image_processor(self) -> FuyuImageProcessor:
76 + return self._get_hf_processor().image_processor
77
78 def _get_image_feature_grid_size(
79 self,
@@ -82,7 +81,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
81 image_width: int,
82 image_height: int,
83 ) -> tuple[int, int]:
85 - target_width, target_height = self._get_image_target_size()
84 + image_processor = self._get_image_processor()
85 + target_width = image_processor.size["width"]
86 + target_height = image_processor.size["height"]
87
88 if not (image_width <= target_width and image_height <= target_height):
89 height_scale_factor = target_height / image_height
@@ -96,8 +97,14 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
97 nrows = math.ceil(image_height / 30)
98 return ncols, nrows
99
100 +
101 +class FuyuProfilingInfo(FuyuProcessingMixin, BaseProfilingInfo):
102 +
103 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104 + return {"image": 1}
105 +
106 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
100 - target_width, target_height = self._get_image_target_size()
107 + target_width, target_height = self._get_image_size_with_most_features()
108
109 max_ncols, max_nrows = self._get_image_feature_grid_size(
110 image_width=target_width,
@@ -107,8 +114,36 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
114
115 return {"image": max_image_tokens}
116
110 - def _get_hf_processor(self) -> FuyuProcessor:
111 - return self.ctx.get_hf_processor(FuyuProcessor)
117 + def _get_image_size_with_most_features(self) -> ImageSize:
118 + image_processor = self._get_image_processor()
119 + return ImageSize(width=image_processor.size["width"],
120 + height=image_processor.size["height"])
121 +
122 + def get_dummy_processor_inputs(
123 + self,
124 + seq_len: int,
125 + mm_counts: Mapping[str, int],
126 + ) -> ProcessorInputs:
127 + target_width, target_height = self._get_image_size_with_most_features()
128 + num_images = mm_counts.get("image", 0)
129 +
130 + mm_data = {
131 + "image":
132 + self._get_dummy_images(width=target_width,
133 + height=target_height,
134 + num_images=num_images)
135 + }
136 +
137 + return ProcessorInputs(
138 + prompt_text="",
139 + mm_data=mm_data,
140 + )
141 +
142 +
143 +class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
144 +
145 + def _get_profiling_info(self) -> BaseProfilingInfo:
146 + return FuyuProfilingInfo(self.ctx)
147
148 def _call_hf_processor(
149 self,
@@ -161,7 +196,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
196 hf_processor_mm_kwargs: Mapping[str, object],
197 out_mm_kwargs: MultiModalKwargs,
198 ) -> list[PromptReplacement]:
164 - hf_config = self.ctx.get_hf_config(FuyuConfig)
199 + hf_config = self._get_hf_config()
200 bos_token_id = hf_config.bos_token_id
201
202 tokenizer = self._get_tokenizer()
@@ -208,26 +243,6 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
243
244 return result
245
211 - def _get_dummy_processor_inputs(
212 - self,
213 - seq_len: int,
214 - mm_counts: Mapping[str, int],
215 - ) -> ProcessorInputs:
216 - target_width, target_height = self._get_image_target_size()
217 - num_images = mm_counts.get("image", 0)
218 -
219 - mm_data = {
220 - "image":
221 - self._get_dummy_images(width=target_width,
222 - height=target_height,
223 - num_images=num_images)
224 - }
225 -
226 - return ProcessorInputs(
227 - prompt_text="",
228 - mm_data=mm_data,
229 - )
230 -
246
247 @MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
248 class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
vllm/model_executor/models/llava.py
+115 -66
@@ -1,4 +1,4 @@
1 -from abc import abstractmethod
1 +from abc import ABC, abstractmethod
2 from functools import cached_property
3 from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4 Protocol, Set, Tuple, TypedDict, Union)
@@ -13,6 +13,7 @@ from transformers.models.pixtral import PixtralProcessor
13
14 from vllm.attention import AttentionMetadata
15 from vllm.config import VllmConfig
16 +from vllm.inputs import InputProcessingContext
17 from vllm.model_executor.layers.activation import get_act_fn
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 RowParallelLinear)
@@ -25,9 +26,10 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
26 NestedTensors)
27 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28 ImageSize)
28 -from vllm.multimodal.processing import (InputProcessingContext,
29 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 MultiModalDataItems, ProcessingCache,
30 - ProcessorInputs, PromptReplacement)
31 + ProcessingMixin, PromptReplacement)
32 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
33 from vllm.sequence import IntermediateTensors
34
35 from .clip import CLIPVisionModel
@@ -37,7 +39,7 @@ from .pixtral import (PixtralHFVisionModel,
39 from .siglip import SiglipVisionModel
40 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
41 maybe_prefix, merge_multimodal_embeddings)
40 -from .vision import BaseVisionLanguageMultiModalProcessor
42 +from .vision import get_vision_encoder_info
43
44
45 class LlavaImagePixelInputs(TypedDict):
@@ -94,30 +96,42 @@ class LlavaMultiModalProjector(nn.Module):
96
97 class LlavaLikeConfig(Protocol):
98 vision_config: Final[PretrainedConfig]
99 + image_token_index: Final[int]
100 vision_feature_select_strategy: Final[str]
98 - vision_feature_layer: Final[Union[int, List[int]]]
101 + vision_feature_layer: Final[Union[int, list[int]]]
102
103
101 -class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
104 +class LlavaLikeProcessor(Protocol):
105 + image_token: Final[str]
106 +
107 +
108 +class BaseLlavaProcessingMixin(ProcessingMixin, ABC):
109
103 - @abstractmethod
110 def _get_hf_config(self) -> LlavaLikeConfig:
105 - raise NotImplementedError
111 + return self.ctx.get_hf_config(LlavaConfig)
112
107 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
108 - return {"image": None}
113 + def _get_vision_encoder_info(self):
114 + return get_vision_encoder_info(self._get_hf_config())
115
110 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
111 - return {"image": self._get_max_image_tokens()}
116 + @abstractmethod
117 + def _get_hf_processor(self) -> LlavaLikeProcessor:
118 + raise NotImplementedError
119
113 - def _get_mm_fields_config(
120 + def _get_num_image_tokens(
121 self,
115 - hf_inputs: BatchFeature,
116 - hf_processor_mm_kwargs: Mapping[str, object],
117 - ) -> Mapping[str, MultiModalFieldConfig]:
118 - return dict(
119 - pixel_values=MultiModalFieldConfig.batched("image"),
120 - image_embeds=MultiModalFieldConfig.batched("image"),
122 + *,
123 + image_width: int,
124 + image_height: int,
125 + ) -> int:
126 + hf_config = self._get_hf_config()
127 + vision_encoder_info = self._get_vision_encoder_info()
128 +
129 + return self._apply_feature_select_strategy(
130 + hf_config.vision_feature_select_strategy,
131 + vision_encoder_info.get_num_image_tokens(
132 + image_width=image_width,
133 + image_height=image_height,
134 + ),
135 )
136
137 def _apply_feature_select_strategy(
@@ -133,31 +147,38 @@ class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
147 msg = f"Unexpected feature select strategy: {strategy!r}"
148 raise NotImplementedError(msg)
149
136 - def _get_max_image_tokens(self) -> int:
137 - hf_config = self._get_hf_config()
150
139 - return self._apply_feature_select_strategy(
140 - hf_config.vision_feature_select_strategy,
141 - self._vision_encoder_info.get_max_image_tokens(),
142 - )
151 +class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
152
144 - def _get_dummy_image_size(self) -> ImageSize:
145 - image_size = self._vision_encoder_info.get_image_size()
146 - return ImageSize(image_size, image_size)
153 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
154 + return {"image": None}
155
148 - @abstractmethod
149 - def _get_image_token(self) -> str:
150 - raise NotImplementedError
156 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
157 + return {"image": self._get_max_image_tokens()}
158 +
159 + def _get_image_size_with_most_features(self) -> ImageSize:
160 + vision_encoder_info = self._get_vision_encoder_info()
161 + width = height = vision_encoder_info.get_image_size()
162 + return ImageSize(width=width, height=height)
163
152 - def _get_dummy_processor_inputs(
164 + def _get_max_image_tokens(self) -> int:
165 + target_width, target_height = self._get_image_size_with_most_features()
166 +
167 + return self._get_num_image_tokens(
168 + image_width=target_width,
169 + image_height=target_height,
170 + )
171 +
172 + def get_dummy_processor_inputs(
173 self,
174 seq_len: int,
175 mm_counts: Mapping[str, int],
176 ) -> ProcessorInputs:
177 num_images = mm_counts.get("image", 0)
178
159 - image_token = self._get_image_token()
160 - target_width, target_height = self._get_dummy_image_size()
179 + processor = self._get_hf_processor()
180 + image_token = processor.image_token
181 + target_width, target_height = self._get_image_size_with_most_features()
182
183 mm_data = {
184 "image":
@@ -172,32 +193,32 @@ class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
193 )
194
195
175 -class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
176 -
177 - def _get_hf_config(self) -> LlavaConfig:
178 - return self.ctx.get_hf_config(LlavaConfig)
196 +class LlavaProcessingMixin(BaseLlavaProcessingMixin):
197
180 - def _get_hf_processor(self) -> LlavaProcessor:
198 + def _get_hf_processor(self):
199 return self.ctx.get_hf_processor(LlavaProcessor)
200
183 - def _get_image_token(self) -> str:
184 - return self._get_hf_processor().image_token
201
186 - def _get_num_image_tokens(
187 - self,
188 - *,
189 - image_width: int,
190 - image_height: int,
191 - ) -> int:
192 - hf_config = self._get_hf_config()
202 +class LlavaProfilingInfo(LlavaProcessingMixin, BaseLlavaProfilingInfo):
203 + pass
204
194 - return self._apply_feature_select_strategy(
195 - hf_config.vision_feature_select_strategy,
196 - self._vision_encoder_info.get_num_image_tokens(
197 - image_width=image_width,
198 - image_height=image_height,
199 - ),
200 - )
205 +
206 +class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
207 + BaseMultiModalProcessor):
208 +
209 + # Copied from BaseMultiModalProcessor
210 + @abstractmethod
211 + def _get_profiling_info(self) -> BaseProfilingInfo:
212 + raise NotImplementedError
213 +
214 + # Copied from BaseMultiModalProcessor
215 + @abstractmethod
216 + def _get_mm_fields_config(
217 + self,
218 + hf_inputs: BatchFeature,
219 + hf_processor_mm_kwargs: Mapping[str, object],
220 + ) -> Mapping[str, MultiModalFieldConfig]:
221 + raise NotImplementedError
222
223 def _get_prompt_replacements(
224 self,
@@ -232,16 +253,37 @@ class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
253 ]
254
255
235 -class PixtralHFMultiModalProcessor(BaseLlavaMultiModalProcessor):
256 +class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
257 +
258 + def _get_profiling_info(self) -> BaseProfilingInfo:
259 + return LlavaProfilingInfo(self.ctx)
260 +
261 + def _get_mm_fields_config(
262 + self,
263 + hf_inputs: BatchFeature,
264 + hf_processor_mm_kwargs: Mapping[str, object],
265 + ) -> Mapping[str, MultiModalFieldConfig]:
266 + return dict(
267 + pixel_values=MultiModalFieldConfig.batched("image"),
268 + image_embeds=MultiModalFieldConfig.batched("image"),
269 + )
270
237 - def _get_hf_config(self) -> LlavaConfig:
238 - return self.ctx.get_hf_config(LlavaConfig)
271
240 - def _get_hf_processor(self) -> PixtralProcessor:
272 +class PixtralHFProcessingMixin(BaseLlavaProcessingMixin):
273 +
274 + def _get_hf_processor(self):
275 return self.ctx.get_hf_processor(PixtralProcessor)
276
243 - def _get_image_token(self) -> str:
244 - return self._get_hf_processor().image_token
277 +
278 +class PixtralHFProfilingInfo(PixtralHFProcessingMixin, BaseLlavaProfilingInfo):
279 + pass
280 +
281 +
282 +class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
283 + BaseMultiModalProcessor):
284 +
285 + def _get_profiling_info(self) -> BaseProfilingInfo:
286 + return PixtralHFProfilingInfo(self.ctx)
287
288 def _call_hf_processor(
289 self,
@@ -270,6 +312,16 @@ class PixtralHFMultiModalProcessor(BaseLlavaMultiModalProcessor):
312
313 return processed_outputs
314
315 + def _get_mm_fields_config(
316 + self,
317 + hf_inputs: BatchFeature,
318 + hf_processor_mm_kwargs: Mapping[str, object],
319 + ) -> Mapping[str, MultiModalFieldConfig]:
320 + return dict(
321 + pixel_values=MultiModalFieldConfig.batched("image"),
322 + image_embeds=MultiModalFieldConfig.batched("image"),
323 + )
324 +
325 def _get_prompt_replacements(
326 self,
327 mm_items: MultiModalDataItems,
@@ -316,7 +368,7 @@ def _build_llava_or_pixtral_hf_processor(
368 *,
369 cache: Optional[ProcessingCache] = None,
370 enable_sanity_checks: bool = True,
319 -) -> BaseLlavaMultiModalProcessor:
371 +) -> BaseMultiModalProcessor:
372 hf_config = ctx.get_hf_config(LlavaConfig)
373
374 if isinstance(hf_config.vision_config, PixtralVisionConfig):
@@ -663,16 +715,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
715
716 class MantisMultiModalProcessor(LlavaMultiModalProcessor):
717
666 - def _get_hf_processor(self):
667 - return self.ctx.get_hf_processor(LlavaProcessor)
668 -
718 def apply(
719 self,
720 prompt_text: str,
721 mm_data: MultiModalDataDict,
722 hf_processor_mm_kwargs: Mapping[str, object],
723 ) -> MultiModalInputsV2:
675 - hf_config = self.ctx.get_hf_config(LlavaConfig)
724 + hf_config = self._get_hf_config()
725 image_token_id = hf_config.image_token_index
726
727 # Assume that it doesn't depend on the image size
vllm/model_executor/models/llava_next.py
+38 -37
@@ -1,6 +1,6 @@
1 from functools import cached_property
2 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
3 - TypedDict, Union)
2 +from typing import (Final, Iterable, List, Literal, Mapping, Optional,
3 + Protocol, Set, Tuple, TypedDict, Union)
4
5 import numpy as np
6 import torch
@@ -17,12 +17,14 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
19 from vllm.multimodal.parse import ImageSize
20 +from vllm.multimodal.profiling import BaseProfilingInfo
21 from vllm.sequence import IntermediateTensors
22
23 from .clip import CLIPVisionModel
24 from .interfaces import SupportsMultiModal, SupportsPP
24 -from .llava import (LlavaMultiModalProcessor, LlavaMultiModalProjector,
25 - init_vision_tower_for_llava)
25 +from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingMixin,
26 + BaseLlavaProfilingInfo, LlavaLikeConfig,
27 + LlavaMultiModalProjector, init_vision_tower_for_llava)
28 from .siglip import SiglipVisionModel
29 from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
30 init_vllm_registered_model, maybe_prefix)
@@ -60,35 +62,17 @@ LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
62 LlavaNextImageEmbeddingInputs]
63
64
63 -class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
65 +class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
66 + image_grid_pinpoints: Final[list[list[int]]]
67
65 - def _get_hf_config(self) -> LlavaNextConfig:
66 - return self.ctx.get_hf_config(LlavaNextConfig)
68
68 - def _get_hf_processor(self) -> LlavaNextProcessor:
69 - return self.ctx.get_hf_processor(LlavaNextProcessor)
69 +class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
70
71 - def _get_mm_fields_config(
72 - self,
73 - hf_inputs: BatchFeature,
74 - hf_processor_mm_kwargs: Mapping[str, object],
75 - ) -> Mapping[str, MultiModalFieldConfig]:
76 - return dict(
77 - pixel_values=MultiModalFieldConfig.batched("image"),
78 - image_sizes=MultiModalFieldConfig.batched("image"),
79 - image_embeds=MultiModalFieldConfig.batched("image"),
80 - )
81 -
82 - def _get_image_token(self) -> str:
83 - return self._get_hf_processor().image_token
84 -
85 - def _get_max_image_tokens(self) -> int:
86 - largest_feature_size, _ = self._get_pinpoint_with_most_features()
87 - return largest_feature_size
71 + def _get_hf_config(self) -> LlavaNextLikeConfig:
72 + return self.ctx.get_hf_config(LlavaNextConfig)
73
89 - def _get_dummy_image_size(self) -> ImageSize:
90 - _, pinpoint = self._get_pinpoint_with_most_features()
91 - return pinpoint
74 + def _get_hf_processor(self):
75 + return self.ctx.get_hf_processor(LlavaNextProcessor)
76
77 # Based on: https://github.com/huggingface/text-generation-inference/blob/v2.2.0/server/text_generation_server/models/vlm_causal_lm.py#L106
78 def _get_num_image_tokens(
@@ -98,7 +82,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
82 image_height: int,
83 ) -> int:
84 hf_config = self._get_hf_config()
101 - vision_encoder_info = self._vision_encoder_info
85 + vision_encoder_info = self._get_vision_encoder_info()
86
87 base_feature_size = self._apply_feature_select_strategy(
88 hf_config.vision_feature_select_strategy,
@@ -140,7 +124,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
124 current_height = npatches * num_patch_height
125 current_width = npatches * num_patch_width
126
143 - # NOTE: HF resizes based on float32
127 + # NOTE: Use float32 to remain consistent with HF output
128 original_aspect_ratio = np.array(original_width / original_height,
129 dtype=np.float32)
130 current_aspect_ratio = np.array(current_width / current_height,
@@ -164,11 +148,10 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
148
149 return (unpadded_features, newline_features)
150
167 - def _get_pinpoint_with_most_features(self) -> tuple[int, ImageSize]:
168 - """
169 - Get the grid pinpoint with the most features and
170 - the corresponding feature size.
171 - """
151 +
152 +class LlavaNextProfilingInfo(LlavaNextProcessingMixin, BaseLlavaProfilingInfo):
153 +
154 + def _get_image_size_with_most_features(self) -> ImageSize:
155 hf_config = self._get_hf_config()
156
157 largest_feature_size, largest_feature_pinpoint = 0, None
@@ -183,7 +166,25 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
166 if largest_feature_size == 0 or largest_feature_pinpoint is None:
167 raise ValueError("Cannot have a largest feature size of 0!")
168
186 - return largest_feature_size, largest_feature_pinpoint
169 + return largest_feature_pinpoint
170 +
171 +
172 +class LlavaNextMultiModalProcessor(LlavaNextProcessingMixin,
173 + BaseLlavaMultiModalProcessor):
174 +
175 + def _get_profiling_info(self) -> BaseProfilingInfo:
176 + return LlavaNextProfilingInfo(self.ctx)
177 +
178 + def _get_mm_fields_config(
179 + self,
180 + hf_inputs: BatchFeature,
181 + hf_processor_mm_kwargs: Mapping[str, object],
182 + ) -> Mapping[str, MultiModalFieldConfig]:
183 + return dict(
184 + pixel_values=MultiModalFieldConfig.batched("image"),
185 + image_sizes=MultiModalFieldConfig.batched("image"),
186 + image_embeds=MultiModalFieldConfig.batched("image"),
187 + )
188
189
190 @MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor)
vllm/model_executor/models/llava_next_video.py
+86 -62
@@ -15,11 +15,14 @@ from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15 from vllm.model_executor.models.clip import CLIPVisionModel
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
19 -from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
20 - VideoEmbeddingItems, VideoProcessorItems)
21 -from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
18 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
19 + NestedTensors)
20 +from vllm.multimodal.parse import (ImageSize, VideoEmbeddingItems,
21 + VideoProcessorItems)
22 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
23 + MultiModalDataItems, ProcessingMixin,
24 PromptReplacement)
25 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
26 from vllm.sequence import IntermediateTensors
27 from vllm.utils import is_list_of
28
@@ -28,7 +31,7 @@ from .llava import init_vision_tower_for_llava
31 from .siglip import SiglipVisionModel
32 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
33 maybe_prefix, merge_multimodal_embeddings)
31 -from .vision import BaseVisionLanguageMultiModalProcessor
34 +from .vision import get_vision_encoder_info
35
36
37 class LlavaNextVideoPixelInputs(TypedDict):
@@ -44,29 +47,16 @@ class LlavaNextVideoPixelInputs(TypedDict):
47 """
48
49
47 -class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
50 +class LlavaNextVideoProcessingMixin(ProcessingMixin):
51
49 - def _get_hf_config(self) -> LlavaNextVideoConfig:
52 + def _get_hf_config(self):
53 return self.ctx.get_hf_config(LlavaNextVideoConfig)
54
52 - def _get_hf_processor(self) -> LlavaNextVideoProcessor:
53 - return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
54 -
55 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
56 - return {"video": 1}
57 -
58 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
59 - num_frames = self._get_dummy_num_frames(seq_len)
60 - max_video_tokens = self._get_max_video_tokens(num_frames)
61 -
62 - return {"video": max_video_tokens}
55 + def _get_vision_encoder_info(self):
56 + return get_vision_encoder_info(self._get_hf_config())
57
64 - def _get_mm_fields_config(
65 - self,
66 - hf_inputs: BatchFeature,
67 - hf_processor_mm_kwargs: Mapping[str, object],
68 - ) -> Mapping[str, MultiModalFieldConfig]:
69 - return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
58 + def _get_hf_processor(self):
59 + return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
60
61 def _get_num_frame_tokens(
62 self,
@@ -77,7 +67,8 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
67 hf_config = self._get_hf_config()
68 spatial_pool_stride = hf_config.spatial_pool_stride
69
80 - patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
70 + vision_encoder_info = self._get_vision_encoder_info()
71 + patch_grid_length = vision_encoder_info.get_patch_grid_length()
72 pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
73
74 return pooled_grid_length * pooled_grid_length
@@ -96,18 +87,43 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
87
88 return num_frame_tokens * num_frames
89
99 - def _get_max_video_tokens(self, num_frames: int) -> int:
100 - return self._get_num_video_tokens(image_width=999999,
101 - image_height=999999,
102 - num_frames=num_frames)
90 +
91 +class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
92 + BaseProfilingInfo):
93 +
94 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
95 + return {"video": 1}
96 +
97 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
98 + target_width, target_height = self._get_image_size_with_most_features()
99 +
100 + max_video_tokens = self._get_num_video_tokens(
101 + image_width=target_width,
102 + image_height=target_height,
103 + num_frames=self._get_dummy_num_frames(seq_len),
104 + )
105 +
106 + return {"video": max_video_tokens}
107 +
108 + def _get_image_size_with_most_features(self) -> ImageSize:
109 + vision_encoder_info = self._get_vision_encoder_info()
110 + width = height = vision_encoder_info.get_image_size()
111 + return ImageSize(width=width, height=height)
112
113 def _get_max_video_frames(self, max_tokens: int) -> int:
114 + target_width, target_height = self._get_image_size_with_most_features()
115 +
116 num_frames = 0
117
118 while True:
119 next_num_frames = num_frames + 1
120 + next_max_tokens = self._get_num_video_tokens(
121 + image_width=target_width,
122 + image_height=target_height,
123 + num_frames=next_num_frames,
124 + )
125
110 - if self._get_max_video_tokens(next_num_frames) > max_tokens:
126 + if next_max_tokens > max_tokens:
127 break
128
129 num_frames = next_num_frames
@@ -122,12 +138,45 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
138
139 return max(max_total_frames // max(max_videos, 1), 1)
140
125 - def _get_dummy_image_size(self) -> ImageSize:
126 - image_size = self._vision_encoder_info.get_image_size()
127 - return ImageSize(image_size, image_size)
141 + def get_dummy_processor_inputs(
142 + self,
143 + seq_len: int,
144 + mm_counts: Mapping[str, int],
145 + ) -> ProcessorInputs:
146 + num_videos = mm_counts.get("video", 0)
147 +
148 + processor = self._get_hf_processor()
149 + video_token = processor.video_token
150 + target_width, target_height = self._get_image_size_with_most_features()
151 +
152 + mm_data = {
153 + "video":
154 + self._get_dummy_videos(
155 + width=target_width,
156 + height=target_height,
157 + num_frames=self._get_dummy_num_frames(seq_len),
158 + num_videos=num_videos,
159 + )
160 + }
161 +
162 + return ProcessorInputs(
163 + prompt_text=video_token * num_videos,
164 + mm_data=mm_data,
165 + )
166 +
167
129 - def _get_video_token(self) -> str:
130 - return self._get_hf_processor().video_token
168 +class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
169 + BaseMultiModalProcessor):
170 +
171 + def _get_profiling_info(self) -> BaseProfilingInfo:
172 + return LlavaNextVideoProfilingInfo(self.ctx)
173 +
174 + def _get_mm_fields_config(
175 + self,
176 + hf_inputs: BatchFeature,
177 + hf_processor_mm_kwargs: Mapping[str, object],
178 + ) -> Mapping[str, MultiModalFieldConfig]:
179 + return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
180
181 def _get_prompt_replacements(
182 self,
@@ -162,36 +211,11 @@ class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
211 ),
212 ]
213
165 - def _get_dummy_processor_inputs(
166 - self,
167 - seq_len: int,
168 - mm_counts: Mapping[str, int],
169 - ) -> ProcessorInputs:
170 - num_videos = mm_counts.get("video", 0)
171 -
172 - video_token = self._get_video_token()
173 - target_width, target_height = self._get_dummy_image_size()
174 -
175 - mm_data = {
176 - "video":
177 - self._get_dummy_videos(
178 - width=target_width,
179 - height=target_height,
180 - num_frames=self._get_dummy_num_frames(seq_len),
181 - num_videos=num_videos,
182 - )
183 - }
184 -
185 - return ProcessorInputs(
186 - prompt_text=video_token * num_videos,
187 - mm_data=mm_data,
188 - )
189 -
214
215 # adopted from transformers modeling_llava_next_video.py
216 class LlavaNextVideoPooler(nn.Module):
217
194 - def __init__(self, config):
218 + def __init__(self, config: LlavaNextVideoConfig):
219 super().__init__()
220
221 mode = config.spatial_pool_mode
@@ -209,7 +233,7 @@ class LlavaNextVideoPooler(nn.Module):
233 raise ValueError(
234 f"Unknown pooling mode: {mode}. Expected [`average`, `max`]")
235
212 - def forward(self, image_features):
236 + def forward(self, image_features: torch.Tensor):
237 ori_width = int(
238 math.sqrt(image_features.shape[1] * self.image_size //
239 self.image_size))
vllm/model_executor/models/llava_onevision.py
+98 -76
@@ -1,7 +1,7 @@
1 import math
2 from functools import cached_property
3 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4 - TypedDict, Union)
3 +from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4 + Protocol, Set, Tuple, TypedDict, Union)
5
6 import numpy as np
7 import torch
@@ -21,15 +21,16 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
22 from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
23 VideoProcessorItems)
24 -from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
25 - PromptReplacement)
24 +from vllm.multimodal.processing import MultiModalFieldConfig, PromptReplacement
25 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
26 from vllm.sequence import IntermediateTensors
27 from vllm.utils import is_list_of
28
29 from .clip import CLIPVisionModel
30 from .interfaces import SupportsMultiModal, SupportsPP
31 -from .llava import init_vision_tower_for_llava
32 -from .llava_next import LlavaNextMultiModalProcessor
31 +from .llava import BaseLlavaProfilingInfo, init_vision_tower_for_llava
32 +from .llava_next import (LlavaNextLikeConfig, LlavaNextMultiModalProcessor,
33 + LlavaNextProcessingMixin)
34 from .siglip import SiglipVisionModel
35 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36 maybe_prefix, merge_multimodal_embeddings)
@@ -82,39 +83,17 @@ LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs,
83 LlavaOnevisionVideoPixelInputs]
84
85
85 -class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
86 +class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
87 + video_token_index: Final[int]
88
87 - def _get_hf_config(self) -> LlavaOnevisionConfig:
88 - return self.ctx.get_hf_config(LlavaOnevisionConfig)
89 -
90 - def _get_hf_processor(self) -> LlavaOnevisionProcessor:
91 - return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
89
93 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
94 - return {"image": None, "video": None}
95 -
96 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
97 - max_image_tokens = self._get_max_image_tokens()
90 +class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
91
99 - num_frames = self._get_dummy_num_frames(seq_len)
100 - max_video_tokens = self._get_max_video_tokens(num_frames)
101 -
102 - return {
103 - "image": max_image_tokens,
104 - "video": max_video_tokens,
105 - }
92 + def _get_hf_config(self) -> LlavaOnevisionLikeConfig:
93 + return self.ctx.get_hf_config(LlavaOnevisionConfig)
94
107 - def _get_mm_fields_config(
108 - self,
109 - hf_inputs: BatchFeature,
110 - hf_processor_mm_kwargs: Mapping[str, object],
111 - ) -> Mapping[str, MultiModalFieldConfig]:
112 - return dict(
113 - pixel_values=MultiModalFieldConfig.batched("image"),
114 - image_sizes=MultiModalFieldConfig.batched("image"),
115 - image_embeds=MultiModalFieldConfig.batched("image"),
116 - pixel_values_videos=MultiModalFieldConfig.batched("video"),
117 - )
95 + def _get_hf_processor(self):
96 + return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
97
98 def _get_num_unpadded_features(
99 self,
@@ -128,7 +107,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
107 current_height = npatches * num_patch_height
108 current_width = npatches * num_patch_width
109
131 - # NOTE: HF resizes based on float32
110 + # NOTE: Use float32 to remain consistent with HF output
111 original_aspect_ratio = np.array(original_width / original_height,
112 dtype=np.float32)
113 current_aspect_ratio = np.array(current_width / current_height,
@@ -167,7 +146,8 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
146 hf_config = self._get_hf_config()
147 spatial_pool_stride = getattr(hf_config, "spatial_pool_stride", 2)
148
170 - patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
149 + vision_encoder_info = self._get_vision_encoder_info()
150 + patch_grid_length = vision_encoder_info.get_patch_grid_length()
151 pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
152
153 return pooled_grid_length * pooled_grid_length
@@ -186,18 +166,33 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
166
167 return num_frame_tokens * num_frames + 1 # Newline token
168
189 - def _get_max_video_tokens(self, num_frames: int) -> int:
190 - return self._get_num_video_tokens(image_width=999999,
191 - image_height=999999,
192 - num_frames=num_frames)
169 +
170 +class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
171 + BaseLlavaProfilingInfo):
172 +
173 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
174 + return {"image": None, "video": None}
175 +
176 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
177 + return {
178 + "image": self._get_max_image_tokens(),
179 + "video": self._get_max_video_tokens(seq_len),
180 + }
181
182 def _get_max_video_frames(self, max_tokens: int) -> int:
183 + target_width, target_height = self._get_image_size_with_most_features()
184 +
185 num_frames = 0
186
187 while True:
188 next_num_frames = num_frames + 1
189 + next_max_tokens = self._get_num_video_tokens(
190 + image_width=target_width,
191 + image_height=target_height,
192 + num_frames=next_num_frames,
193 + )
194
200 - if self._get_max_video_tokens(next_num_frames) > max_tokens:
195 + if next_max_tokens > max_tokens:
196 break
197
198 num_frames = next_num_frames
@@ -215,8 +210,65 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
210
211 return max(max_total_frames // max(max_videos, 1), 1)
212
218 - def _get_video_token(self) -> str:
219 - return self._get_hf_processor().video_token
213 + def _get_max_video_tokens(self, seq_len: int) -> int:
214 + target_width, target_height = self._get_image_size_with_most_features()
215 +
216 + return self._get_num_video_tokens(
217 + image_width=target_width,
218 + image_height=target_height,
219 + num_frames=self._get_dummy_num_frames(seq_len),
220 + )
221 +
222 + def get_dummy_processor_inputs(
223 + self,
224 + seq_len: int,
225 + mm_counts: Mapping[str, int],
226 + ) -> ProcessorInputs:
227 + num_images = mm_counts.get("image", 0)
228 + num_videos = mm_counts.get("video", 0)
229 +
230 + processor = self._get_hf_processor()
231 + image_token = processor.image_token
232 + video_token = processor.video_token
233 + target_width, target_height = self._get_image_size_with_most_features()
234 +
235 + mm_data = {
236 + "image":
237 + self._get_dummy_images(width=target_width,
238 + height=target_height,
239 + num_images=num_images),
240 + "video":
241 + self._get_dummy_videos(
242 + width=target_width,
243 + height=target_height,
244 + num_frames=self._get_dummy_num_frames(seq_len),
245 + num_videos=num_videos,
246 + )
247 + }
248 +
249 + return ProcessorInputs(
250 + prompt_text=image_token * num_images + video_token * num_videos,
251 + mm_data=mm_data,
252 + )
253 +
254 +
255 +class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
256 + LlavaNextMultiModalProcessor):
257 +
258 + def _get_profiling_info(self) -> BaseProfilingInfo:
259 + return LlavaOnevisionProfilingInfo(self.ctx)
260 +
261 + def _get_mm_fields_config(
262 + self,
263 + hf_inputs: BatchFeature,
264 + hf_processor_mm_kwargs: Mapping[str, object],
265 + ) -> Mapping[str, MultiModalFieldConfig]:
266 + return dict(
267 + pixel_values=MultiModalFieldConfig.batched("image"),
268 + image_sizes=MultiModalFieldConfig.batched("image"),
269 + image_embeds=MultiModalFieldConfig.batched("image"),
270 + pixel_values_videos=MultiModalFieldConfig.batched("video"),
271 + )
272
273 def _call_hf_processor(
274 self,
@@ -235,7 +287,8 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
287 mm_kwargs=mm_kwargs,
288 )
289
238 - video_token = self._get_video_token()
290 + processor = self._get_hf_processor()
291 + video_token = processor.video_token
292
293 # LLaVA-OneVision processor doesn't support multiple videos
294 # with different sizes when converting back to tensors
@@ -303,37 +356,6 @@ class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
356 ),
357 ]
358
306 - def _get_dummy_processor_inputs(
307 - self,
308 - seq_len: int,
309 - mm_counts: Mapping[str, int],
310 - ) -> ProcessorInputs:
311 - num_images = mm_counts.get("image", 0)
312 - num_videos = mm_counts.get("video", 0)
313 -
314 - image_token = self._get_image_token()
315 - video_token = self._get_video_token()
316 - target_width, target_height = self._get_dummy_image_size()
317 -
318 - mm_data = {
319 - "image":
320 - self._get_dummy_images(width=target_width,
321 - height=target_height,
322 - num_images=num_images),
323 - "video":
324 - self._get_dummy_videos(
325 - width=target_width,
326 - height=target_height,
327 - num_frames=self._get_dummy_num_frames(seq_len),
328 - num_videos=num_videos,
329 - )
330 - }
331 -
332 - return ProcessorInputs(
333 - prompt_text=image_token * num_images + video_token * num_videos,
334 - mm_data=mm_data,
335 - )
336 -
359
360 class LlavaOnevisionMultiModalProjector(nn.Module):
361
vllm/model_executor/models/phi3v.py
+59 -45
@@ -14,7 +14,7 @@
14 # limitations under the License.
15 from collections.abc import Iterable, Mapping, Sequence
16 from functools import cached_property
17 -from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
17 +from typing import Any, List, Literal, Optional, Set, Tuple, TypedDict, Union
18
19 import torch
20 import torch.nn as nn
@@ -28,22 +28,23 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
28 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
29 from vllm.model_executor.layers.vocab_parallel_embedding import (
30 VocabParallelEmbedding)
31 -from vllm.model_executor.models.clip import CLIPVisionModel
31 from vllm.model_executor.sampling_metadata import SamplingMetadata
32 from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 MultiModalInputsV2, MultiModalKwargs,
35 NestedTensors, PlaceholderRange)
37 -from vllm.multimodal.parse import ImageEmbeddingItems, ImageProcessorItems
36 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
37 + ImageSize)
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 - MultiModalDataItems, ProcessorInputs,
39 + MultiModalDataItems, ProcessingMixin,
40 PromptReplacement,
41 _BoundPromptReplacement,
42 _PlaceholderInfo)
43 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
44 from vllm.sequence import IntermediateTensors
45 from vllm.utils import is_list_of
46
46 -from .clip import dummy_image_for_clip
47 +from .clip import CLIPVisionModel
48 from .interfaces import SupportsMultiModal, SupportsPP
49 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
50 init_vllm_registered_model, maybe_prefix,
@@ -54,10 +55,6 @@ logger = init_logger(__name__)
55 # Cannot find the following 2 numbers from hf config.
56 _IMAGE_TOKEN_ID = 32044
57
57 -# Result in the max possible feature size (h:w = 16:1)
58 -MAX_IMAGE_FEATURE_SIZE_HEIGHT = 8000
59 -MAX_IMAGE_FEATURE_SIZE_WIDTH = 50
60 -
58 CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
59 hidden_act="quick_gelu",
60 hidden_size=1024,
@@ -305,10 +302,17 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
302 return image_features_hd_newline
303
304
308 -class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
305 +class Phi3VProcessingMixin(ProcessingMixin):
306
310 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
311 - return {"image": None}
307 + def _get_hf_processor(
308 + self,
309 + *,
310 + num_crops: Optional[int] = None,
311 + ) -> ProcessorMixin:
312 + if num_crops is not None:
313 + return self.ctx.get_hf_processor(num_crops=num_crops)
314 +
315 + return self.ctx.get_hf_processor()
316
317 def _get_num_image_tokens(
318 self,
@@ -323,23 +327,55 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
327 height=image_height,
328 )
329
330 +
331 +class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
332 +
333 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
334 + return {"image": None}
335 +
336 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
337 + target_width, target_height = self._get_image_size_with_most_features()
338 +
339 max_image_tokens = self._get_num_image_tokens(
328 - image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
329 - image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
340 + image_width=target_width,
341 + image_height=target_height,
342 )
343
344 return {"image": max_image_tokens}
345
334 - def _get_hf_processor(
346 + def _get_image_size_with_most_features(self) -> ImageSize:
347 + # Result in the max possible feature size (h:w = 16:1)
348 + return ImageSize(height=8000, width=50)
349 +
350 + def get_dummy_processor_inputs(
351 self,
336 - *,
337 - num_crops: Optional[int] = None,
338 - ) -> ProcessorMixin:
339 - if num_crops is not None:
340 - return self.ctx.get_hf_processor(num_crops=num_crops)
352 + seq_len: int,
353 + mm_counts: Mapping[str, int],
354 + ) -> ProcessorInputs:
355 + num_images = mm_counts.get("image", 0)
356
342 - return self.ctx.get_hf_processor()
357 + target_width, target_height = self._get_image_size_with_most_features()
358 +
359 + mm_data = {
360 + "image":
361 + self._get_dummy_images(width=target_width,
362 + height=target_height,
363 + num_images=num_images)
364 + }
365 +
366 + hf_processor = self._get_hf_processor()
367 + image_tokens: list[str] = hf_processor.img_tokens # type: ignore
368 +
369 + return ProcessorInputs(
370 + prompt_text="".join(image_tokens[:num_images]),
371 + mm_data=mm_data,
372 + )
373 +
374 +
375 +class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
376 +
377 + def _get_profiling_info(self) -> BaseProfilingInfo:
378 + return Phi3VProfilingInfo(self.ctx)
379
380 def _call_hf_processor(
381 self,
@@ -377,10 +413,10 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
413 def _get_prompt_replacements(
414 self,
415 mm_items: MultiModalDataItems,
380 - hf_processor_mm_kwargs: Mapping[str, object],
416 + hf_processor_mm_kwargs: Mapping[str, Any],
417 out_mm_kwargs: MultiModalKwargs,
418 ) -> list[PromptReplacement]:
383 - hf_processor = self._get_hf_processor()
419 + hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
420 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
421
422 tokenizer = self._get_tokenizer()
@@ -442,28 +478,6 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
478
479 return token_ids, text, placeholders
480
445 - def _get_dummy_processor_inputs(
446 - self,
447 - seq_len: int,
448 - mm_counts: Mapping[str, int],
449 - ) -> ProcessorInputs:
450 - num_images = mm_counts.get("image", 0)
451 -
452 - data = dummy_image_for_clip(
453 - CLIP_VIT_LARGE_PATCH14_336_CONFIG,
454 - num_images,
455 - image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
456 - image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
457 - )
458 -
459 - hf_processor = self._get_hf_processor()
460 - image_tokens: list[str] = hf_processor.img_tokens # type: ignore
461 -
462 - return ProcessorInputs(
463 - prompt_text="".join(image_tokens[:num_images]),
464 - mm_data=data,
465 - )
466 -
481 def apply(
482 self,
483 prompt_text: str,
vllm/model_executor/models/qwen2_audio.py
+59 -37
@@ -20,8 +20,8 @@
20 # limitations under the License.
21 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
22 from functools import cached_property
23 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24 - Union)
23 +from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
24 + TypedDict, Union)
25
26 import torch
27 import torch.nn as nn
@@ -40,8 +40,9 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40 NestedTensors)
41 from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser
42 from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 - MultiModalDataItems, ProcessorInputs,
43 + MultiModalDataItems, ProcessingMixin,
44 PromptReplacement)
45 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
46 from vllm.sequence import IntermediateTensors
47
48 from .interfaces import SupportsMultiModal, SupportsPP
@@ -79,28 +80,70 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
80 return feat_lengths, output_lengths
81
82
82 -class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
83 +class Qwen2AudioProcessingMixin(ProcessingMixin):
84 +
85 + def _get_hf_config(self):
86 + return self.ctx.get_hf_config(Qwen2AudioConfig)
87 +
88 + def _get_hf_processor(
89 + self,
90 + *,
91 + # Ignored in initialization
92 + sampling_rate: Optional[int] = None,
93 + ) -> Qwen2AudioProcessor:
94 + return self.ctx.get_hf_processor(Qwen2AudioProcessor)
95 +
96 + def _get_feature_extractor(
97 + self,
98 + *,
99 + # Ignored in initialization
100 + sampling_rate: Optional[int] = None,
101 + ) -> WhisperFeatureExtractor:
102 + hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
103 + feature_extractor = hf_processor.feature_extractor # type: ignore
104 + assert isinstance(feature_extractor, WhisperFeatureExtractor)
105 + return feature_extractor
106 +
107 +
108 +class Qwen2AudioProfilingInfo(Qwen2AudioProcessingMixin, BaseProfilingInfo):
109
110 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
111 return {"audio": None}
112
113 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
88 - hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
114 + hf_config = self._get_hf_config()
115 max_source_positions = hf_config.audio_config.max_source_positions
116 max_output_lengths = (max_source_positions - 2) // 2 + 1
117
118 return {"audio": max_output_lengths}
119
94 - def _get_hf_processor(
120 + def get_dummy_processor_inputs(
121 self,
96 - *,
97 - # Ignored in initialization
98 - sampling_rate: Optional[int] = None,
99 - ) -> Qwen2AudioProcessor:
100 - return self.ctx.get_hf_processor(Qwen2AudioProcessor)
122 + seq_len: int,
123 + mm_counts: Mapping[str, int],
124 + ) -> ProcessorInputs:
125 + feature_extractor = self._get_feature_extractor()
126 +
127 + sampling_rate = feature_extractor.sampling_rate
128 + audio_len = feature_extractor.chunk_length * sampling_rate
129 + num_audios = mm_counts.get("audio", 0)
130 +
131 + mm_data = {
132 + "audio":
133 + self._get_dummy_audios(length=audio_len, num_audios=num_audios)
134 + }
135 +
136 + return ProcessorInputs(
137 + prompt_text="<|AUDIO|>" * num_audios,
138 + mm_data=mm_data,
139 + )
140 +
141
102 - def _get_feature_extractor(self) -> WhisperFeatureExtractor:
103 - return self._get_hf_processor().feature_extractor # type: ignore
142 +class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
143 + BaseMultiModalProcessor):
144 +
145 + def _get_profiling_info(self) -> BaseProfilingInfo:
146 + return Qwen2AudioProfilingInfo(self.ctx)
147
148 def _get_data_parser(self) -> MultiModalDataParser:
149 feature_extractor = self._get_feature_extractor()
@@ -110,7 +153,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
153 self,
154 prompt: str,
155 mm_data: Mapping[str, object],
113 - mm_kwargs: Mapping[str, object],
156 + mm_kwargs: Mapping[str, Any],
157 ) -> BatchFeature:
158 mm_data = dict(mm_data)
159 audios = mm_data.pop("audios", [])
@@ -118,7 +161,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
161 if audios:
162 mm_data["audios"] = audios
163
121 - feature_extractor = self._get_feature_extractor()
164 + feature_extractor = self._get_feature_extractor(**mm_kwargs)
165 mm_kwargs = dict(
166 **mm_kwargs,
167 sampling_rate=feature_extractor.sampling_rate,
@@ -151,7 +194,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
194 hf_processor_mm_kwargs: Mapping[str, object],
195 out_mm_kwargs: MultiModalKwargs,
196 ) -> list[PromptReplacement]:
154 - hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
197 + hf_config = self._get_hf_config()
198 placeholder = hf_config.audio_token_index
199
200 feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
@@ -191,27 +234,6 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
234 # tokens than the number of audio items)
235 return True
236
194 - def _get_dummy_processor_inputs(
195 - self,
196 - seq_len: int,
197 - mm_counts: Mapping[str, int],
198 - ) -> ProcessorInputs:
199 - feature_extractor = self._get_feature_extractor()
200 -
201 - sampling_rate = feature_extractor.sampling_rate
202 - audio_len = feature_extractor.chunk_length * sampling_rate
203 - num_audios = mm_counts.get("audio", 0)
204 -
205 - mm_data = {
206 - "audio":
207 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)
208 - }
209 -
210 - return ProcessorInputs(
211 - prompt_text="<|AUDIO|>" * num_audios,
212 - mm_data=mm_data,
213 - )
214 -
237
238 @MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
239 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
vllm/model_executor/models/qwen2_vl.py
+144 -87
@@ -59,8 +59,9 @@ from vllm.multimodal.inputs import (ImageItem, ModalityData,
59 from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60 MultiModalDataParser)
61 from vllm.multimodal.processing import (BaseMultiModalProcessor,
62 - MultiModalDataItems, ProcessorInputs,
62 + MultiModalDataItems, ProcessingMixin,
63 PromptReplacement)
64 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
65 from vllm.platforms import _Backend
66 from vllm.sequence import IntermediateTensors
67 from vllm.transformers_utils.config import uses_mrope
@@ -708,10 +709,44 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
709 return super()._parse_video_data(data)
710
711
711 -class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
712 +class Qwen2VLProcessingMixin(ProcessingMixin):
713
713 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
714 - return {"image": None, "video": None}
714 + def _get_hf_config(self):
715 + return self.ctx.get_hf_config(Qwen2VLConfig)
716 +
717 + def _get_hf_processor(
718 + self,
719 + *,
720 + min_pixels: Optional[int] = None,
721 + max_pixels: Optional[int] = None,
722 + ) -> Qwen2VLProcessor:
723 + hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
724 + image_processor = hf_processor.image_processor # type: ignore
725 + assert isinstance(image_processor, Qwen2VLImageProcessor)
726 +
727 + if min_pixels:
728 + image_processor.min_pixels = min_pixels
729 + if max_pixels:
730 + image_processor.max_pixels = max_pixels
731 + if max_pixels or min_pixels:
732 + image_processor.size = {
733 + "min_pixels": image_processor.min_pixels,
734 + "max_pixels": image_processor.max_pixels,
735 + }
736 +
737 + return hf_processor
738 +
739 + def _get_image_processor(
740 + self,
741 + *,
742 + min_pixels: Optional[int] = None,
743 + max_pixels: Optional[int] = None,
744 + ):
745 + hf_processor = self._get_hf_processor(min_pixels=min_pixels,
746 + max_pixels=max_pixels)
747 + image_processor = hf_processor.image_processor # type: ignore
748 + assert isinstance(image_processor, Qwen2VLImageProcessor)
749 + return image_processor
750
751 def _get_vision_info(
752 self,
@@ -721,14 +756,13 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
756 num_frames: int = 1,
757 do_resize: bool = True,
758 ) -> tuple[ImageSize, int]:
724 - hf_config = self.ctx.get_hf_config(Qwen2VLConfig)
759 + hf_config = self._get_hf_config()
760 vision_config = hf_config.vision_config
761 patch_size = vision_config.patch_size
762 merge_size = vision_config.spatial_merge_size
763 temporal_patch_size = vision_config.temporal_patch_size
764
730 - hf_processor = self._get_hf_processor()
731 - image_processor = self._get_image_processor(hf_processor)
765 + image_processor = self._get_image_processor()
766
767 if do_resize:
768 resized_height, resized_width = smart_resize(
@@ -753,7 +787,45 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
787
788 return preprocessed_size, num_vision_tokens
789
756 - def _get_dummy_image_size(self) -> ImageSize:
790 + def _get_num_image_tokens(
791 + self,
792 + *,
793 + image_width: int,
794 + image_height: int,
795 + ) -> int:
796 + _, num_image_tokens = self._get_vision_info(
797 + image_width=image_width,
798 + image_height=image_height,
799 + )
800 + return num_image_tokens
801 +
802 + def _get_num_video_tokens(
803 + self,
804 + *,
805 + image_width: int,
806 + image_height: int,
807 + num_frames: int,
808 + ) -> int:
809 + _, num_video_tokens = self._get_vision_info(
810 + image_width=image_width,
811 + image_height=image_height,
812 + num_frames=num_frames,
813 + )
814 + return num_video_tokens
815 +
816 +
817 +class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
818 +
819 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
820 + return {"image": None, "video": None}
821 +
822 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
823 + return {
824 + "image": self._get_max_image_tokens(),
825 + "video": self._get_max_video_tokens(seq_len),
826 + }
827 +
828 + def _get_image_size_with_most_features(self) -> ImageSize:
829 max_image_size, _ = self._get_vision_info(
830 image_width=9999999,
831 image_height=9999999,
@@ -761,27 +833,27 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
833 return max_image_size
834
835 def _get_max_image_tokens(self) -> int:
764 - _, max_image_tokens = self._get_vision_info(
765 - image_width=9999999,
766 - image_height=9999999,
767 - )
768 - return max_image_tokens
836 + target_width, target_height = self._get_image_size_with_most_features()
837
770 - def _get_max_video_tokens(self, num_frames: int) -> int:
771 - _, max_video_tokens = self._get_vision_info(
772 - image_width=9999999,
773 - image_height=9999999,
774 - num_frames=num_frames,
838 + return self._get_num_image_tokens(
839 + image_width=target_width,
840 + image_height=target_height,
841 )
776 - return max_video_tokens
842
843 def _get_max_video_frames(self, max_tokens: int) -> int:
844 + target_width, target_height = self._get_image_size_with_most_features()
845 +
846 num_frames = 0
847
848 while True:
849 next_num_frames = num_frames + 1
850 + next_max_tokens = self._get_num_video_tokens(
851 + image_width=target_width,
852 + image_height=target_height,
853 + num_frames=next_num_frames,
854 + )
855
784 - if self._get_max_video_tokens(next_num_frames) > max_tokens:
856 + if next_max_tokens > max_tokens:
857 break
858
859 num_frames = next_num_frames
@@ -797,56 +869,73 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
869 max_total_frames = self._get_max_video_frames(seq_len -
870 max_image_tokens)
871
800 - return max(max_total_frames // max(max_videos, 1), 1)
872 + num_frames = max(max_total_frames // max(max_videos, 1), 1)
873
802 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
803 - max_image_tokens = self._get_max_image_tokens()
874 + # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
875 + if num_frames > 1 and num_frames % 2 == 1:
876 + num_frames += 1
877
805 - num_frames = self._get_dummy_num_frames(seq_len)
806 - max_video_tokens = self._get_max_video_tokens(num_frames)
878 + return num_frames
879
808 - return {
809 - "image": max_image_tokens,
810 - "video": max_video_tokens,
880 + def _get_max_video_tokens(self, seq_len: int) -> int:
881 + target_width, target_height = self._get_image_size_with_most_features()
882 +
883 + return self._get_num_video_tokens(
884 + image_width=target_width,
885 + image_height=target_height,
886 + num_frames=self._get_dummy_num_frames(seq_len),
887 + )
888 +
889 + def get_dummy_processor_inputs(
890 + self,
891 + seq_len: int,
892 + mm_counts: Mapping[str, int],
893 + ) -> ProcessorInputs:
894 + num_images = mm_counts.get("image", 0)
895 + num_videos = mm_counts.get("video", 0)
896 +
897 + hf_processor = self._get_hf_processor()
898 + image_token: str = hf_processor.image_token
899 + video_token: str = hf_processor.video_token
900 + target_width, target_height = self._get_image_size_with_most_features()
901 +
902 + mm_data = {
903 + "image":
904 + self._get_dummy_images(width=target_width,
905 + height=target_height,
906 + num_images=num_images),
907 + "video":
908 + self._get_dummy_videos(
909 + width=target_width,
910 + height=target_height,
911 + num_frames=self._get_dummy_num_frames(seq_len),
912 + num_videos=num_videos,
913 + )
914 }
915
813 - def _get_data_parser(self) -> MultiModalDataParser:
814 - return Qwen2MultiModalDataParser()
916 + return ProcessorInputs(
917 + prompt_text=image_token * num_images + video_token * num_videos,
918 + mm_data=mm_data,
919 + )
920
816 - def _get_image_processor(self, hf_processor: Qwen2VLProcessor):
817 - image_processor = hf_processor.image_processor # type: ignore
818 - assert isinstance(image_processor, Qwen2VLImageProcessor)
819 - return image_processor
921
821 - def _get_hf_processor(
822 - self,
823 - *,
824 - min_pixels: Optional[int] = None,
825 - max_pixels: Optional[int] = None,
826 - ) -> Qwen2VLProcessor:
827 - hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
828 - image_processor = self._get_image_processor(hf_processor)
922 +class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
923 + BaseMultiModalProcessor):
924
830 - if min_pixels:
831 - image_processor.min_pixels = min_pixels
832 - if max_pixels:
833 - image_processor.max_pixels = max_pixels
834 - if max_pixels or min_pixels:
835 - image_processor.size = {
836 - "min_pixels": image_processor.min_pixels,
837 - "max_pixels": image_processor.max_pixels,
838 - }
925 + def _get_profiling_info(self) -> BaseProfilingInfo:
926 + return Qwen2VLProfilingInfo(self.ctx)
927
840 - return hf_processor
928 + def _get_data_parser(self) -> MultiModalDataParser:
929 + return Qwen2MultiModalDataParser()
930
931 def _get_prompt_replacements(
932 self,
933 mm_items: MultiModalDataItems,
845 - hf_processor_mm_kwargs: Mapping[str, object],
934 + hf_processor_mm_kwargs: Mapping[str, Any],
935 out_mm_kwargs: MultiModalKwargs,
936 ) -> list[PromptReplacement]:
848 - hf_processor = self._get_hf_processor()
849 - image_processor = self._get_image_processor(hf_processor)
937 + hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
938 + image_processor = self._get_image_processor(**hf_processor_mm_kwargs)
939
940 # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
941 # image_token and video_token registered
@@ -901,38 +990,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
990 video_grid_thw=MultiModalFieldConfig.batched("video"),
991 )
992
904 - def _get_dummy_processor_inputs(
905 - self,
906 - seq_len: int,
907 - mm_counts: Mapping[str, int],
908 - ) -> ProcessorInputs:
909 - num_images = mm_counts.get("image", 0)
910 - num_videos = mm_counts.get("video", 0)
911 -
912 - hf_processor = self._get_hf_processor()
913 - image_token: str = hf_processor.image_token
914 - video_token: str = hf_processor.video_token
915 - target_width, target_height = self._get_dummy_image_size()
916 -
917 - mm_data = {
918 - "image":
919 - self._get_dummy_images(width=target_width,
920 - height=target_height,
921 - num_images=num_images),
922 - "video":
923 - self._get_dummy_videos(
924 - width=target_width,
925 - height=target_height,
926 - num_frames=self._get_dummy_num_frames(seq_len),
927 - num_videos=num_videos,
928 - )
929 - }
930 -
931 - return ProcessorInputs(
932 - prompt_text=image_token * num_images + video_token * num_videos,
933 - mm_data=mm_data,
934 - )
935 -
993
994 @MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
995 class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
vllm/model_executor/models/ultravox.py
+55 -36
@@ -3,8 +3,8 @@
3
4 import math
5 from functools import cached_property
6 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7 - TypedDict, Union)
6 +from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
7 + Tuple, TypedDict, Union)
8
9 import torch
10 import torch.utils.checkpoint
@@ -26,8 +26,9 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
26 NestedTensors)
27 from vllm.multimodal.parse import MultiModalDataParser
28 from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 - MultiModalDataItems, ProcessorInputs,
29 + MultiModalDataItems, ProcessingMixin,
30 PromptReplacement)
31 +from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
32 from vllm.sequence import IntermediateTensors
33 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
34
@@ -55,7 +56,30 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
56 UltravoxAudioEmbeddingInputs]
57
58
58 -class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
59 +class UltravoxProcessingMixin(ProcessingMixin):
60 +
61 + def _get_hf_processor(
62 + self,
63 + *,
64 + # Ignored in initialization
65 + sampling_rate: Optional[int] = None,
66 + ) -> ProcessorMixin:
67 + return self.ctx.get_hf_processor()
68 +
69 + def _get_feature_extractor(
70 + self,
71 + *,
72 + # Ignored in initialization
73 + sampling_rate: Optional[int] = None,
74 + ) -> WhisperFeatureExtractor:
75 + hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
76 + audio_processor = hf_processor.audio_processor # type: ignore
77 + feature_extractor = audio_processor.feature_extractor # type: ignore
78 + assert isinstance(feature_extractor, WhisperFeatureExtractor)
79 + return feature_extractor
80 +
81 +
82 +class UltravoxProfilingInfo(UltravoxProcessingMixin, BaseProfilingInfo):
83
84 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
85 return {"audio": None}
@@ -67,17 +91,33 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
91
92 return {"audio": max_audio_tokens}
93
70 - def _get_hf_processor(
94 + def get_dummy_processor_inputs(
95 self,
72 - *,
73 - # Ignored in initialization
74 - sampling_rate: Optional[int] = None,
75 - ) -> ProcessorMixin:
76 - return self.ctx.get_hf_processor()
96 + seq_len: int,
97 + mm_counts: Mapping[str, int],
98 + ) -> ProcessorInputs:
99 + feature_extractor = self._get_feature_extractor()
100 +
101 + sampling_rate = feature_extractor.sampling_rate
102 + audio_len = feature_extractor.chunk_length * sampling_rate
103 + num_audios = mm_counts.get("audio", 0)
104 +
105 + mm_data = {
106 + "audio":
107 + self._get_dummy_audios(length=audio_len, num_audios=num_audios)
108 + }
109 +
110 + return ProcessorInputs(
111 + prompt_text="<|audio|>" * num_audios,
112 + mm_data=mm_data,
113 + )
114
78 - def _get_feature_extractor(self) -> WhisperFeatureExtractor:
79 - hf_processor = self._get_hf_processor()
80 - return hf_processor.audio_processor.feature_extractor # type: ignore
115 +
116 +class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
117 + BaseMultiModalProcessor):
118 +
119 + def _get_profiling_info(self) -> BaseProfilingInfo:
120 + return UltravoxProfilingInfo(self.ctx)
121
122 def _get_data_parser(self) -> MultiModalDataParser:
123 feature_extractor = self._get_feature_extractor()
@@ -155,10 +195,10 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
195 def _get_prompt_replacements(
196 self,
197 mm_items: MultiModalDataItems,
158 - hf_processor_mm_kwargs: Mapping[str, object],
198 + hf_processor_mm_kwargs: Mapping[str, Any],
199 out_mm_kwargs: MultiModalKwargs,
200 ) -> list[PromptReplacement]:
161 - hf_processor = self._get_hf_processor()
201 + hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
202 placeholder = hf_processor.audio_token_replacement # type: ignore
203
204 def get_replacement_ultravox(item_idx: int):
@@ -173,27 +213,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
213 )
214 ]
215
176 - def _get_dummy_processor_inputs(
177 - self,
178 - seq_len: int,
179 - mm_counts: Mapping[str, int],
180 - ) -> ProcessorInputs:
181 - feature_extractor = self._get_feature_extractor()
182 -
183 - sampling_rate = feature_extractor.sampling_rate
184 - audio_len = feature_extractor.chunk_length * sampling_rate
185 - num_audios = mm_counts.get("audio", 0)
186 -
187 - mm_data = {
188 - "audio":
189 - self._get_dummy_audios(length=audio_len, num_audios=num_audios)
190 - }
191 -
192 - return ProcessorInputs(
193 - prompt_text="<|audio|>" * num_audios,
194 - mm_data=mm_data,
195 - )
196 -
216
217 class StackAudioFrames(nn.Module):
218 """
vllm/model_executor/models/vision.py
+8 -29
@@ -1,12 +1,8 @@
1 from abc import ABC, abstractmethod
2 -from typing import Final, Generic, Optional, Protocol, TypeVar
2 +from typing import Final, Generic, Protocol, TypeVar
3
4 from transformers import PretrainedConfig
5
6 -from vllm.multimodal.processing import (BaseMultiModalProcessor,
7 - InputProcessingContext,
8 - ProcessingCache)
9 -
6 _C = TypeVar("_C", bound=PretrainedConfig)
7
8
@@ -43,12 +39,18 @@ class VisionEncoderInfo(ABC, Generic[_C]):
39 raise NotImplementedError
40
41
46 -def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
42 +class VisionLanguageConfig(Protocol):
43 + vision_config: Final[PretrainedConfig]
44 +
45 +
46 +def get_vision_encoder_info(
47 + hf_config: VisionLanguageConfig) -> VisionEncoderInfo:
48 # Avoid circular imports
49 from .clip import CLIPEncoderInfo, CLIPVisionConfig
50 from .pixtral import PixtralHFEncoderInfo, PixtralVisionConfig
51 from .siglip import SiglipEncoderInfo, SiglipVisionConfig
52
53 + vision_config = hf_config.vision_config
54 if isinstance(vision_config, CLIPVisionConfig):
55 return CLIPEncoderInfo(vision_config)
56 if isinstance(vision_config, PixtralVisionConfig):
@@ -58,26 +60,3 @@ def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
60
61 msg = f"Unsupported vision config: {type(vision_config)}"
62 raise NotImplementedError(msg)
61 -
62 -
63 -class VisionLanguageConfig(Protocol):
64 - vision_config: Final[PretrainedConfig]
65 -
66 -
67 -class BaseVisionLanguageMultiModalProcessor(BaseMultiModalProcessor):
68 -
69 - def __init__(self,
70 - ctx: InputProcessingContext,
71 - *,
72 - cache: Optional[ProcessingCache] = None,
73 - enable_sanity_checks: bool = True) -> None:
74 - super().__init__(ctx,
75 - cache=cache,
76 - enable_sanity_checks=enable_sanity_checks)
77 -
78 - vision_config = self._get_hf_config().vision_config
79 - self._vision_encoder_info = vision_encoder_info(vision_config)
80 -
81 - @abstractmethod
82 - def _get_hf_config(self) -> VisionLanguageConfig:
83 - raise NotImplementedError
vllm/multimodal/processing.py
+43 -109
@@ -8,11 +8,10 @@ from functools import lru_cache
8 from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
9
10 import numpy as np
11 -import numpy.typing as npt
11 import torch
12 from blake3 import blake3
13 from PIL import Image
15 -from transformers import BatchFeature, ProcessorMixin
14 +from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
15
16 from vllm.inputs import DummyData, InputProcessingContext
17 from vllm.logger import init_logger
@@ -24,6 +23,7 @@ from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
23 MultiModalInputsV2, MultiModalKwargs,
24 MultiModalKwargsItem, PlaceholderRange)
25 from .parse import MultiModalDataItems, MultiModalDataParser
26 +from .profiling import BaseProfilingInfo
27
28 logger = init_logger(__name__)
29
@@ -466,14 +466,6 @@ def find_mm_placeholders(
466 return dict(full_groupby_modality(it))
467
468
469 -@dataclass
470 -class ProcessorInputs:
471 - """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
472 - prompt_text: str
473 - mm_data: MultiModalDataDict
474 - hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
475 -
476 -
469 class ProcessingCache:
470
471 def __init__(self, capacity: int) -> None:
@@ -585,9 +577,33 @@ class ProcessingCache:
577 self._cache.put(cache_key, output_kwargs)
578
579
588 -class BaseMultiModalProcessor(ABC):
580 +class ProcessingMixin:
581 + """
582 + Contains helper functions to perform processing.
583 +
584 + Not to be confused with :class:`transformers.ProcessorMixin`.
585 + """
586 + ctx: InputProcessingContext
587 +
588 + def _get_tokenizer(self) -> AnyTokenizer:
589 + return self.ctx.tokenizer
590 +
591 + def _get_hf_config(self) -> PretrainedConfig:
592 + return self.ctx.get_hf_config()
593 +
594 + def _get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
595 + """
596 + Subclasses can override this method to handle
597 + specific kwargs from model config or user inputs.
598 + """
599 + return self.ctx.get_hf_processor(**kwargs)
600 +
601 +
602 +class BaseMultiModalProcessor(ProcessingMixin, ABC):
603 """
604 Abstract base class to process multi-modal inputs to be used in vLLM.
605 +
606 + Not to be confused with :class:`transformers.ProcessorMixin`.
607 """
608
609 def __init__(self,
@@ -601,6 +617,9 @@ class BaseMultiModalProcessor(ABC):
617 self.cache = cache
618 self.enable_sanity_checks = enable_sanity_checks
619
620 + self.data_parser = self._get_data_parser()
621 + self.profiling_info = self._get_profiling_info()
622 +
623 def __call__(
624 self,
625 prompt: str,
@@ -609,32 +628,9 @@ class BaseMultiModalProcessor(ABC):
628 ) -> MultiModalInputsV2:
629 return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
630
612 - @abstractmethod
613 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
614 - """
615 - Return the maximum supported number of items for each modality.
616 -
617 - A value of `None` means unlimited number of items.
618 -
619 - Omitting a modality from the returned dictionary means that
620 - it is not supported at all.
621 - """
622 - raise NotImplementedError
623 -
624 - @abstractmethod
625 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
626 - """
627 - Get the maximum possible number of tokens per data item
628 - for each modality.
629 -
630 - The dictionary returned by this method should have the same
631 - keys as that returned by :meth:`get_supported_mm_limits`.
632 - """
633 - raise NotImplementedError
634 -
631 def _get_data_parser(self) -> MultiModalDataParser:
632 """
637 - Construct a data parser to preprocess multi-modal data items
633 + Construct a parser to preprocess multi-modal data items
634 before passing them to :meth:`_get_hf_mm_data`.
635
636 You can support additional modalities by creating a subclass
@@ -642,15 +638,12 @@ class BaseMultiModalProcessor(ABC):
638 """
639 return MultiModalDataParser()
640
645 - def _get_hf_processor(self) -> ProcessorMixin:
641 + def _get_profiling_info(self) -> BaseProfilingInfo:
642 """
647 - Subclasses can add keyword arguments to this method to accept
648 - additional kwargs from model config or user inputs.
643 + Get the profiling information to find the worst-case memory usage of
644 + the model.
645 """
650 - return self.ctx.get_hf_processor()
651 -
652 - def _get_tokenizer(self) -> AnyTokenizer:
653 - return self.ctx.tokenizer
646 + raise NotImplementedError
647
648 def _to_mm_items(
649 self,
@@ -660,8 +653,7 @@ class BaseMultiModalProcessor(ABC):
653 Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
654 before passing them to :meth:`_get_hf_mm_data`.
655 """
663 - parser = self._get_data_parser()
664 - mm_items = parser.parse_mm_data(mm_data)
656 + mm_items = self.data_parser.parse_mm_data(mm_data)
657
658 mm_limits = self.ctx.get_mm_config().limit_per_prompt
659 for modality, items in mm_items.items():
@@ -799,7 +791,7 @@ class BaseMultiModalProcessor(ABC):
791
792 # Some HF processors (e.g. Qwen2-VL) expect corresponding
793 # multi-modal tokens to be in the prompt text
802 - dummy_inputs = self._get_dummy_processor_inputs(
794 + dummy_inputs = self.profiling_info.get_dummy_processor_inputs(
795 self.ctx.model_config.max_model_len,
796 mm_missing_counts,
797 )
@@ -1133,73 +1125,14 @@ class BaseMultiModalProcessor(ABC):
1125 mm_placeholders=mm_placeholder_ranges,
1126 )
1127
1136 - def _get_dummy_audios(
1137 - self,
1138 - *,
1139 - length: int,
1140 - num_audios: int,
1141 - ) -> list[npt.NDArray]:
1142 - audio = np.zeros((length, ))
1143 - return [audio] * num_audios
1144 -
1145 - def _get_dummy_images(
1146 - self,
1147 - *,
1148 - width: int,
1149 - height: int,
1150 - num_images: int,
1151 - ) -> list[Image.Image]:
1152 - image = Image.new("RGB", (width, height), color=0)
1153 - return [image] * num_images
1154 -
1155 - def _get_dummy_videos(
1156 - self,
1157 - *,
1158 - width: int,
1159 - height: int,
1160 - num_frames: int,
1161 - num_videos: int,
1162 - ) -> list[npt.NDArray]:
1163 - video = np.zeros((num_frames, width, height, 3))
1164 - return [video] * num_videos
1165 -
1166 - @abstractmethod
1167 - def _get_dummy_processor_inputs(
1168 - self,
1169 - seq_len: int,
1170 - mm_counts: Mapping[str, int],
1171 - ) -> ProcessorInputs:
1172 - """
1173 - Build the multi-modal portion of the input which, after processing,
1174 - results in `mm_max_tokens` in :meth:`get_dummy_data`.
1175 - """
1176 - raise NotImplementedError
1177 -
1178 - def _get_and_validate_dummy_mm_counts(self) -> Mapping[str, int]:
1179 - mm_limit_per_prompt = self.ctx.get_mm_config().limit_per_prompt
1180 - supported_mm_limits = self.get_supported_mm_limits()
1181 -
1182 - mm_limits = {
1183 - modality: mm_limit_per_prompt.get(modality, 1)
1184 - for modality in supported_mm_limits
1185 - }
1186 -
1187 - for modality, supported_limit in supported_mm_limits.items():
1188 - limit = mm_limits[modality]
1189 - if supported_limit is not None and supported_limit < limit:
1190 - raise ValueError(
1191 - f"You set {modality}={limit} (or defaulted to 1) in "
1192 - f"`--limit-mm-per-prompt`, but this model only supports "
1193 - f"at most {supported_limit} {modality} items.")
1194 -
1195 - return mm_limits
1196 -
1128 def _get_dummy_mm_inputs(
1129 self,
1130 seq_len: int,
1131 mm_counts: Mapping[str, int],
1132 ) -> MultiModalInputsV2:
1202 - processor_inputs = self._get_dummy_processor_inputs(seq_len, mm_counts)
1133 + profiling = self.profiling_info
1134 + processor_inputs = profiling.get_dummy_processor_inputs(
1135 + seq_len, mm_counts)
1136
1137 return self.apply(
1138 prompt_text=processor_inputs.prompt_text,
@@ -1211,8 +1144,9 @@ class BaseMultiModalProcessor(ABC):
1144 # Avoid circular import
1145 from vllm.sequence import SequenceData
1146
1214 - mm_counts = self._get_and_validate_dummy_mm_counts()
1215 - mm_max_tokens_per_item = self.get_mm_max_tokens_per_item(seq_len)
1147 + profiling = self.profiling_info
1148 + mm_counts = profiling.get_mm_limits()
1149 + mm_max_tokens_per_item = profiling.get_mm_max_tokens_per_item(seq_len)
1150 if mm_counts.keys() != mm_max_tokens_per_item.keys():
1151 raise AssertionError(
1152 "The keys returned by `get_supported_mm_limits`"
vllm/multimodal/profiling.py new
+121
@@ -0,0 +1,121 @@
1 +from abc import ABC, abstractmethod
2 +from collections.abc import Mapping
3 +from dataclasses import dataclass, field
4 +from typing import Optional
5 +
6 +import numpy as np
7 +import numpy.typing as npt
8 +from PIL import Image
9 +
10 +from vllm.inputs import InputProcessingContext
11 +from vllm.logger import init_logger
12 +
13 +from .inputs import MultiModalDataDict
14 +
15 +logger = init_logger(__name__)
16 +
17 +
18 +@dataclass
19 +class ProcessorInputs:
20 + """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
21 + prompt_text: str
22 + mm_data: MultiModalDataDict
23 + hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
24 +
25 +
26 +class BaseProfilingInfo(ABC):
27 + """
28 + Abstract base class that provides the information necessary to profile
29 + multi-modal models.
30 + """
31 +
32 + def __init__(self, ctx: InputProcessingContext) -> None:
33 + super().__init__()
34 +
35 + self.ctx = ctx
36 +
37 + @abstractmethod
38 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
39 + """
40 + Return the maximum supported number of items for each modality.
41 +
42 + A value of `None` means unlimited number of items.
43 +
44 + Omitting a modality from the returned dictionary means that
45 + it is not supported at all.
46 + """
47 + raise NotImplementedError
48 +
49 + @abstractmethod
50 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
51 + """
52 + Get the maximum possible number of tokens per data item
53 + for each modality.
54 +
55 + The dictionary returned by this method should have the same
56 + keys as that returned by :meth:`get_supported_mm_limits`.
57 + """
58 + raise NotImplementedError
59 +
60 + @abstractmethod
61 + def get_dummy_processor_inputs(
62 + self,
63 + seq_len: int,
64 + mm_counts: Mapping[str, int],
65 + ) -> ProcessorInputs:
66 + """
67 + Build the multi-modal portion of the input which, after processing,
68 + results in `mm_max_tokens` in :meth:`get_mm_max_tokens_per_item`.
69 + """
70 + raise NotImplementedError
71 +
72 + def _get_dummy_audios(
73 + self,
74 + *,
75 + length: int,
76 + num_audios: int,
77 + ) -> list[npt.NDArray]:
78 + audio = np.zeros((length, ))
79 + return [audio] * num_audios
80 +
81 + def _get_dummy_images(
82 + self,
83 + *,
84 + width: int,
85 + height: int,
86 + num_images: int,
87 + ) -> list[Image.Image]:
88 + image = Image.new("RGB", (width, height), color=0)
89 + return [image] * num_images
90 +
91 + def _get_dummy_videos(
92 + self,
93 + *,
94 + width: int,
95 + height: int,
96 + num_frames: int,
97 + num_videos: int,
98 + ) -> list[npt.NDArray]:
99 + video = np.zeros((num_frames, width, height, 3))
100 + return [video] * num_videos
101 +
102 + def get_mm_limits(self) -> Mapping[str, int]:
103 + mm_config = self.ctx.get_mm_config()
104 + mm_limit_per_prompt = mm_config.limit_per_prompt
105 +
106 + supported_mm_limits = self.get_supported_mm_limits()
107 +
108 + mm_limits = {
109 + modality: mm_limit_per_prompt.get(modality, 1)
110 + for modality in supported_mm_limits
111 + }
112 +
113 + for modality, supported_limit in supported_mm_limits.items():
114 + limit = mm_limits[modality]
115 + if supported_limit is not None and supported_limit < limit:
116 + raise ValueError(
117 + f"You set {modality}={limit} (or defaulted to 1) in "
118 + f"`--limit-mm-per-prompt`, but this model only supports "
119 + f"at most {supported_limit} {modality} items.")
120 +
121 + return mm_limits
vllm/multimodal/registry.py
+1 -1
@@ -224,7 +224,7 @@ class MultiModalRegistry:
224 tokenizer = cached_get_tokenizer(model_config.tokenizer)
225 processor = self.create_processor(model_config, tokenizer)
226 seq_len = model_config.max_model_len
227 - return processor.get_mm_max_tokens_per_item(seq_len)
227 + return processor.profiling_info.get_mm_max_tokens_per_item(seq_len)
228
229 return {
230 key: plugin.get_max_multimodal_tokens(model_config)