@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2a0596bc48
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Reorganize profiling/processing-related code (#11812)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 8, 2025 at 18:59 UTC 2a0596bc480bb835dc05a30f5e708ecbfffbcd69
23 files changed +833 -760
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+14 -27
@@ -4,24 +4,17 @@ from functools import partial
4 import pytest
5 from PIL import Image
6 from pqdm.threads import pqdm
7 -from transformers import AutoTokenizer
7
9 -from vllm.inputs import InputProcessingContext
8 +from vllm.multimodal import MULTIMODAL_REGISTRY
9 from vllm.multimodal.parse import ImageSize
10 +from vllm.multimodal.processing import BaseMultiModalProcessor
11 +from vllm.multimodal.utils import cached_get_tokenizer
12
13 from ....utils import build_model_context
14
15
15 -# Fixtures lazy import to avoid initializing CUDA during test collection
16 -@pytest.fixture()
17 -def processor_for_llava_next():
18 - from vllm.model_executor.models.llava_next import (
19 - LlavaNextMultiModalProcessor)
20 - return LlavaNextMultiModalProcessor
21 -
22 -
16 def _validate_image_prompt_replacements_one(
24 - processor,
17 + processor: BaseMultiModalProcessor,
18 num_imgs: int,
19 failed_size_excs: list[tuple[ImageSize, Exception]],
20 image_size: ImageSize,
@@ -78,20 +71,17 @@ def _test_image_prompt_replacements(
71
72 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
73 @pytest.mark.parametrize("num_imgs", [1, 2])
81 -def test_processor_prompt_replacements_regression(
82 - processor_for_llava_next,
83 - model_id: str,
84 - num_imgs: int,
85 -):
74 +def test_processor_prompt_replacements_regression(model_id, num_imgs):
75 ctx = build_model_context(
76 model_name=model_id,
77 tokenizer_name=model_id,
78 mm_processor_kwargs=None,
79 limit_mm_per_prompt={"image": num_imgs},
80 )
92 - tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
93 - ctx = InputProcessingContext(ctx.model_config, tokenizer)
94 - processor = processor_for_llava_next(ctx)
81 + processor = MULTIMODAL_REGISTRY.create_processor(
82 + ctx.model_config,
83 + tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
84 + )
85
86 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
87 (488, 183), (2560, 1669)]
@@ -111,20 +101,17 @@ def test_processor_prompt_replacements_regression(
101 "Comment this out to run it manually.")
102 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
103 @pytest.mark.parametrize("num_imgs", [1])
114 -def test_processor_prompt_replacements_all(
115 - processor_for_llava_next,
116 - model_id: str,
117 - num_imgs: int,
118 -):
104 +def test_processor_prompt_replacements_all(model_id, num_imgs):
105 ctx = build_model_context(
106 model_name=model_id,
107 tokenizer_name=model_id,
108 mm_processor_kwargs=None,
109 limit_mm_per_prompt={"image": num_imgs},
110 )
125 - tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126 - ctx = InputProcessingContext(ctx.model_config, tokenizer)
127 - processor = processor_for_llava_next(ctx)
111 + processor = MULTIMODAL_REGISTRY.create_processor(
112 + ctx.model_config,
113 + tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
114 + )
115
116 seen_aspect_ratios = set[float]()
117 image_sizes = list[ImageSize]()
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py
+14 -27
@@ -4,24 +4,17 @@ from functools import partial
4 import pytest
5 from PIL import Image
6 from pqdm.threads import pqdm
7 -from transformers import AutoTokenizer
7
9 -from vllm.inputs import InputProcessingContext
8 +from vllm.multimodal import MULTIMODAL_REGISTRY
9 from vllm.multimodal.parse import ImageSize
10 +from vllm.multimodal.processing import BaseMultiModalProcessor
11 +from vllm.multimodal.utils import cached_get_tokenizer
12
13 from ....utils import build_model_context
14
15
15 -# Fixtures lazy import to avoid initializing CUDA during test collection
16 -@pytest.fixture()
17 -def processor_for_llava_onevision():
18 - from vllm.model_executor.models.llava_onevision import (
19 - LlavaOnevisionMultiModalProcessor)
20 - return LlavaOnevisionMultiModalProcessor
21 -
22 -
16 def _validate_image_prompt_replacements_one(
24 - processor,
17 + processor: BaseMultiModalProcessor,
18 num_imgs: int,
19 failed_size_excs: list[tuple[ImageSize, Exception]],
20 image_size: ImageSize,
@@ -77,20 +70,17 @@ def _test_image_prompt_replacements(
70 @pytest.mark.parametrize("model_id",
71 ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
72 @pytest.mark.parametrize("num_imgs", [1, 2])
80 -def test_processor_prompt_replacements_regression(
81 - processor_for_llava_onevision,
82 - model_id: str,
83 - num_imgs: int,
84 -):
73 +def test_processor_prompt_replacements_regression(model_id, num_imgs):
74 ctx = build_model_context(
75 model_name=model_id,
76 tokenizer_name=model_id,
77 mm_processor_kwargs=None,
78 limit_mm_per_prompt={"image": num_imgs},
79 )
91 - tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
92 - ctx = InputProcessingContext(ctx.model_config, tokenizer)
93 - processor = processor_for_llava_onevision(ctx)
80 + processor = MULTIMODAL_REGISTRY.create_processor(
81 + ctx.model_config,
82 + tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
83 + )
84
85 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
86 (488, 183), (2560, 1669)]
@@ -111,20 +101,17 @@ def test_processor_prompt_replacements_regression(
101 @pytest.mark.parametrize("model_id",
102 ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
103 @pytest.mark.parametrize("num_imgs", [1])
114 -def test_processor_prompt_replacements_all(
115 - processor_for_llava_onevision,
116 - model_id: str,
117 - num_imgs: int,
118 -):
104 +def test_processor_prompt_replacements_all(model_id, num_imgs):
105 ctx = build_model_context(
106 model_name=model_id,
107 tokenizer_name=model_id,
108 mm_processor_kwargs=None,
109 limit_mm_per_prompt={"image": num_imgs},
110 )
125 - tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126 - ctx = InputProcessingContext(ctx.model_config, tokenizer)
127 - processor = processor_for_llava_onevision(ctx)
111 + processor = MULTIMODAL_REGISTRY.create_processor(
112 + ctx.model_config,
113 + tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
114 + )
115
116 seen_aspect_ratios = set[float]()
117 image_sizes = list[ImageSize]()
tests/models/decoder_only/vision_language/processing/test_phi3v.py
+10 -14
@@ -1,21 +1,13 @@
1 """Tests for phi3v's multimodal preprocessing kwargs."""
2 import pytest
3 -from transformers import AutoTokenizer
3
5 -from vllm.inputs import InputProcessingContext
6 -from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
4 +from vllm.multimodal import MULTIMODAL_REGISTRY
5 +from vllm.multimodal.utils import cached_get_tokenizer
6
7 from .....conftest import _ImageAssets
8 from ....utils import build_model_context
9
10
12 -# Wrap lazy imports to avoid initializing CUDA during test collection
13 -@pytest.fixture()
14 -def processor_for_phi3v():
15 - from vllm.model_executor.models.phi3v import Phi3VMultiModalProcessor
16 - return Phi3VMultiModalProcessor
17 -
18 -
11 @pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
12 # yapf: disable
13 @pytest.mark.parametrize(
@@ -29,7 +21,6 @@ def processor_for_phi3v():
21 # yapf: enable
22 @pytest.mark.parametrize("num_imgs", [1, 2])
23 def test_processor_override(
32 - processor_for_phi3v,
24 image_assets: _ImageAssets,
25 model_id: str,
26 mm_processor_kwargs: dict[str, int],
@@ -37,21 +28,26 @@ def test_processor_override(
28 num_imgs: int,
29 ):
30 """Ensure input_processor_for_phi3v handles num_crops properly."""
31 + # Avoid initializing CUDA early
32 + from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
33 +
34 ctx = build_model_context(
35 model_name=model_id,
36 tokenizer_name=model_id,
37 trust_remote_code=True,
38 limit_mm_per_prompt={"image": num_imgs},
39 )
46 - tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
47 - ctx = InputProcessingContext(ctx.model_config, tokenizer)
40 + tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
41 + processor = MULTIMODAL_REGISTRY.create_processor(
42 + ctx.model_config,
43 + tokenizer=tokenizer,
44 + )
45
46 # Build the image str / prompt based on the number of images we pass
47 img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
48 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
49 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
50
54 - processor = processor_for_phi3v(ctx)
51 processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
52
53 # Ensure we have the right number of placeholders per num_crops size
tests/models/decoder_only/vision_language/processing/test_qwen2_vl.py
+8 -14
@@ -1,19 +1,12 @@
1 import pytest
2 -from transformers import AutoTokenizer
2
4 -from vllm.inputs import InputProcessingContext
3 +from vllm.multimodal import MULTIMODAL_REGISTRY
4 +from vllm.multimodal.utils import cached_get_tokenizer
5
6 from .....conftest import _ImageAssets
7 from ....utils import build_model_context
8
9
10 -# Fixtures lazy import to avoid initializing CUDA during test collection
11 -@pytest.fixture()
12 -def processor_for_qwen2_vl():
13 - from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalProcessor
14 - return Qwen2VLMultiModalProcessor
15 -
16 -
10 @pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
11 # yapf: disable
12 @pytest.mark.parametrize(
@@ -24,7 +17,6 @@ def processor_for_qwen2_vl():
17 # yapf: enable
18 @pytest.mark.parametrize("num_imgs", [1, 2])
19 def test_processor_override(
27 - processor_for_qwen2_vl,
20 image_assets: _ImageAssets,
21 model_id: str,
22 mm_processor_kwargs: dict[str, object],
@@ -39,18 +31,20 @@ def test_processor_override(
31 mm_processor_kwargs=None,
32 limit_mm_per_prompt={"image": num_imgs},
33 )
42 - tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
43 - ctx = InputProcessingContext(ctx.model_config, tokenizer)
34 + tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
35 + processor = MULTIMODAL_REGISTRY.create_processor(
36 + ctx.model_config,
37 + tokenizer=tokenizer,
38 + )
39
40 # Build the image str / prompt based on the number of images we pass
41 prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
42 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
43
49 - processor = processor_for_qwen2_vl(ctx)
44 processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
45
46 # Ensure we have the right number of placeholders per num_crops size
53 - hf_processor = processor._get_hf_processor(**mm_processor_kwargs)
47 + hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
48 image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
49 img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
50 pixel_shape = processed_inputs["mm_kwargs"]["pixel_values"].shape
tests/multimodal/test_processing.py
+25 -27
@@ -10,12 +10,17 @@ from PIL import Image
10 from vllm.config import ModelConfig
11 from vllm.inputs import InputProcessingContext
12 from vllm.multimodal import MULTIMODAL_REGISTRY
13 -from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
14 - _PlaceholderInfo, find_mm_placeholders,
13 +# yapf conflicts with isort for this block
14 +# yapf: disable
15 +from vllm.multimodal.processing import (PlaceholderInfo, ProcessingCache,
16 + PromptReplacement,
17 + find_mm_placeholders,
18 find_text_matches, find_token_matches,
19 iter_token_matches,
20 replace_text_matches,
21 replace_token_matches)
22 +# yapf: enable
23 +from vllm.multimodal.profiling import MultiModalProfiler
24 from vllm.multimodal.utils import cached_get_tokenizer
25 from vllm.transformers_utils.tokenizer import AnyTokenizer
26 from vllm.utils import full_groupby
@@ -431,7 +436,7 @@ def test_find_replace_tokens(
436 [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
437 {
438 "pattern_1": [
434 - _PlaceholderInfo(
439 + PlaceholderInfo(
440 modality="pattern_1",
441 item_idx=0,
442 start_idx=6,
@@ -445,13 +450,13 @@ def test_find_replace_tokens(
450 [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
451 {
452 "pattern_1": [
448 - _PlaceholderInfo(
453 + PlaceholderInfo(
454 modality="pattern_1",
455 item_idx=0,
456 start_idx=1,
457 replacement=[32000, 32000],
458 ),
454 - _PlaceholderInfo(
459 + PlaceholderInfo(
460 modality="pattern_1",
461 item_idx=1,
462 start_idx=5,
@@ -459,7 +464,7 @@ def test_find_replace_tokens(
464 ),
465 ],
466 "pattern_3": [
462 - _PlaceholderInfo(
467 + PlaceholderInfo(
468 modality="pattern_3",
469 item_idx=0,
470 start_idx=7,
@@ -472,13 +477,13 @@ def test_find_replace_tokens(
477 [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
478 {
479 "pattern_1": [
475 - _PlaceholderInfo(
480 + PlaceholderInfo(
481 modality="pattern_1",
482 item_idx=0,
483 start_idx=1,
484 replacement=[32000, 32000],
485 ),
481 - _PlaceholderInfo(
486 + PlaceholderInfo(
487 modality="pattern_1",
488 item_idx=1,
489 start_idx=3,
@@ -486,7 +491,7 @@ def test_find_replace_tokens(
491 ),
492 ],
493 "pattern_3": [
489 - _PlaceholderInfo(
494 + PlaceholderInfo(
495 modality="pattern_3",
496 item_idx=0,
497 start_idx=6,
@@ -577,19 +582,15 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
582 revision=None,
583 limit_mm_per_prompt=limit_mm_per_prompt,
584 )
580 - model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
585
582 - processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
583 - ctx = InputProcessingContext(
586 + processor = MULTIMODAL_REGISTRY.create_processor(
587 model_config,
588 tokenizer=cached_get_tokenizer(model_config.tokenizer),
589 )
587 -
588 - processor = processor_factory(ctx, cache=None)
589 - profiler = processor.profiling_info
590 + profiler = MultiModalProfiler(processor)
591
592 mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
592 - profiler.get_supported_mm_limits = mock_supported_mm_limits
593 + processor.info.get_supported_mm_limits = mock_supported_mm_limits
594
595 if is_valid:
596 exc_ctx = nullcontext()
@@ -597,7 +598,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
598 exc_ctx = pytest.raises(ValueError, match="this model only supports")
599
600 with exc_ctx:
600 - profiler.get_mm_limits()
601 + profiler.get_dummy_data(model_config.max_model_len)
602
603
604 @pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@@ -620,16 +621,12 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
621 revision=None,
622 limit_mm_per_prompt=limit_mm_per_prompt,
623 )
623 - model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
624
625 - processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
626 - ctx = InputProcessingContext(
625 + processor = MULTIMODAL_REGISTRY.create_processor(
626 model_config,
627 tokenizer=cached_get_tokenizer(model_config.tokenizer),
628 )
629
631 - processor = processor_factory(ctx, cache=None)
632 -
630 rng = np.random.RandomState(0)
631 image = _rand_img(rng, min_wh=128, max_wh=256)
632 if num_images == 0:
@@ -681,9 +678,9 @@ def _test_processing_cache_correctness(
678 hf_overrides=hf_overrides,
679 limit_mm_per_prompt=limit_mm_per_prompt,
680 )
684 - model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
681
686 - processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
682 + model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
683 + factories = MULTIMODAL_REGISTRY._processor_factories[model_cls]
684 ctx = InputProcessingContext(
685 model_config,
686 tokenizer=cached_get_tokenizer(model_config.tokenizer),
@@ -691,8 +688,9 @@ def _test_processing_cache_correctness(
688 # Ensure that it can fit all of the data
689 cache = ProcessingCache(capacity=1 << 30)
690
694 - baseline_processor = processor_factory(ctx, cache=None)
695 - cached_processor = processor_factory(ctx, cache=cache)
691 + baseline_processor = factories.build_processor(ctx, cache=None)
692 + cached_processor = factories.build_processor(ctx, cache=cache)
693 + dummy_inputs = baseline_processor.dummy_inputs
694
695 rng = np.random.RandomState(0)
696
@@ -724,7 +722,7 @@ def _test_processing_cache_correctness(
722 }
723
724 mm_counts = {k: len(vs) for k, vs in mm_data.items()}
727 - prompt = baseline_processor.profiling_info.get_dummy_processor_inputs(
725 + prompt = dummy_inputs.get_dummy_processor_inputs(
726 model_config.max_model_len,
727 mm_counts,
728 ).prompt_text
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+7 -3
@@ -2,13 +2,17 @@ from typing import Optional
2
3 import torch
4
5 -from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6 - LlavaMultiModalProcessor)
5 +from vllm.model_executor.models.llava import (LlavaDummyInputsBuilder,
6 + LlavaForConditionalGeneration,
7 + LlavaMultiModalProcessor,
8 + LlavaProcessingInfo)
9 from vllm.model_executor.sampling_metadata import SamplingMetadata
10 from vllm.multimodal import MULTIMODAL_REGISTRY
11
12
11 -@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
13 +@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor,
14 + info=LlavaProcessingInfo,
15 + dummy_inputs=LlavaDummyInputsBuilder)
16 class MyLlava(LlavaForConditionalGeneration):
17
18 def compute_logits(
vllm/inputs/preprocess.py
+1 -1
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
9 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10 -from vllm.multimodal.processing import MultiModalDataDict, MultiModalInputsV2
10 +from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputsV2
11 from vllm.prompt_adapter.request import PromptAdapterRequest
12 from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
13 from vllm.utils import print_info_once, print_warning_once
vllm/inputs/registry.py
+3 -1
@@ -323,6 +323,7 @@ class InputRegistry:
323 # Avoid circular import
324 from vllm.model_executor.model_loader import get_model_architecture
325 from vllm.multimodal import MultiModalKwargs
326 + from vllm.multimodal.profiling import MultiModalProfiler
327 from vllm.multimodal.utils import cached_get_tokenizer
328
329 if mm_registry.has_processor(model_config):
@@ -331,7 +332,8 @@ class InputRegistry:
332 trust_remote_code=model_config.trust_remote_code,
333 )
334 processor = mm_registry.create_processor(model_config, tokenizer)
334 - dummy_data = processor.get_dummy_data(seq_len)
335 + profiler = MultiModalProfiler(processor)
336 + dummy_data = profiler.get_dummy_data(seq_len)
337 else:
338 model_cls, _ = get_model_architecture(model_config)
339 if is_encoder_data:
vllm/model_executor/models/aria.py
+23 -24
@@ -23,10 +23,10 @@ from vllm.model_executor.model_loader.weight_utils import (
23 from vllm.multimodal import MULTIMODAL_REGISTRY
24 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
25 NestedTensors)
26 +from vllm.multimodal.parse import MultiModalDataItems
27 from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 - MultiModalDataItems, ProcessingMixin,
28 - PromptReplacement)
29 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
28 + BaseProcessingInfo, PromptReplacement)
29 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30 from vllm.sequence import IntermediateTensors
31 from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
32 AriaVisionConfig)
@@ -445,33 +445,33 @@ def build_mm_projector(config: PretrainedConfig):
445 )
446
447
448 -class AriaProcessingMixin(ProcessingMixin):
448 +class AriaProcessingInfo(BaseProcessingInfo):
449
450 - def _get_hf_config(self):
450 + def get_hf_config(self):
451 return self.ctx.get_hf_config()
452
453 - def _get_vision_config(self) -> AriaVisionConfig:
454 - return self._get_hf_config().vision_config
455 -
456 - def _get_num_image_tokens(self) -> int:
457 - hf_config = self._get_hf_config()
458 - return max(hf_config.projector_patch_to_query_dict.values())
459 -
460 -
461 -class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
453 + def get_vision_config(self) -> AriaVisionConfig:
454 + return self.get_hf_config().vision_config
455
456 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
457 return {"image": None}
458
459 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
467 - return {"image": self._get_num_image_tokens()}
460 + return {"image": self.get_num_image_tokens()}
461 +
462 + def get_num_image_tokens(self) -> int:
463 + hf_config = self.get_hf_config()
464 + return max(hf_config.projector_patch_to_query_dict.values())
465 +
466 +
467 +class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
468
469 def get_dummy_processor_inputs(
470 self,
471 seq_len: int,
472 mm_counts: Mapping[str, int],
473 ) -> ProcessorInputs:
474 - vision_config = self._get_vision_config()
474 + vision_config = self.info.get_vision_config()
475
476 max_image_size = vision_config.image_size
477 num_images = mm_counts.get("image", 0)
@@ -483,7 +483,7 @@ class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
483 num_images=num_images)
484 }
485
486 - hf_processor = self._get_hf_processor()
486 + hf_processor = self.info.get_hf_processor()
487 image_token: str = hf_processor.image_token # type: ignore
488
489 return ProcessorInputs(
@@ -492,10 +492,7 @@ class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
492 )
493
494
495 -class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
496 -
497 - def _get_profiling_info(self) -> BaseProfilingInfo:
498 - return AriaProfilingInfo(self.ctx)
495 +class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
496
497 def _get_mm_fields_config(
498 self,
@@ -513,10 +510,10 @@ class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
510 hf_processor_mm_kwargs: Mapping[str, object],
511 out_mm_kwargs: MultiModalKwargs,
512 ) -> list[PromptReplacement]:
516 - hf_config = self._get_hf_config()
513 + hf_config = self.info.get_hf_config()
514 image_token_id = hf_config.image_token_index
515
519 - num_image_tokens = self._get_num_image_tokens()
516 + num_image_tokens = self.info.get_num_image_tokens()
517
518 return [
519 PromptReplacement(
@@ -527,7 +524,9 @@ class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
524 ]
525
526
530 -@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
527 +@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor,
528 + info=AriaProcessingInfo,
529 + dummy_inputs=AriaDummyInputsBuilder)
530 class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
531 """
532 Aria model for conditional generation tasks.
vllm/model_executor/models/blip2.py
+19 -20
@@ -17,10 +17,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
17 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18 MultiModalInputsV2, MultiModalKwargs,
19 NestedTensors, PlaceholderRange)
20 +from vllm.multimodal.parse import MultiModalDataItems
21 from vllm.multimodal.processing import (BaseMultiModalProcessor,
21 - MultiModalDataItems, ProcessingMixin,
22 - PromptReplacement)
23 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
22 + BaseProcessingInfo, PromptReplacement)
23 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
24 from vllm.sequence import IntermediateTensors
25
26 from .blip import BlipVisionModel
@@ -397,30 +397,30 @@ class Blip2QFormerModel(nn.Module):
397 return sequence_output
398
399
400 -class Blip2ProcessingMixin(ProcessingMixin):
400 +class Blip2ProcessingInfo(BaseProcessingInfo):
401
402 - def _get_hf_config(self):
402 + def get_hf_config(self):
403 return self.ctx.get_hf_config(Blip2Config)
404
405 - def _get_num_image_tokens(self) -> int:
406 - hf_config = self._get_hf_config()
407 - return hf_config.num_query_tokens
408 -
409 -
410 -class Blip2ProfilingInfo(Blip2ProcessingMixin, BaseProfilingInfo):
411 -
405 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
406 return {"image": 1}
407
408 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
416 - return {"image": self._get_num_image_tokens()}
409 + return {"image": self.get_num_image_tokens()}
410 +
411 + def get_num_image_tokens(self) -> int:
412 + hf_config = self.get_hf_config()
413 + return hf_config.num_query_tokens
414 +
415 +
416 +class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
417
418 def get_dummy_processor_inputs(
419 self,
420 seq_len: int,
421 mm_counts: Mapping[str, int],
422 ) -> ProcessorInputs:
423 - hf_config = self._get_hf_config()
423 + hf_config = self.info.get_hf_config()
424 vision_config = hf_config.vision_config
425
426 max_image_size = vision_config.image_size
@@ -439,10 +439,7 @@ class Blip2ProfilingInfo(Blip2ProcessingMixin, BaseProfilingInfo):
439 )
440
441
442 -class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
443 -
444 - def _get_profiling_info(self) -> BaseProfilingInfo:
445 - return Blip2ProfilingInfo(self.ctx)
442 +class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
443
444 def _get_mm_fields_config(
445 self,
@@ -460,7 +457,7 @@ class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
457 hf_processor_mm_kwargs: Mapping[str, object],
458 out_mm_kwargs: MultiModalKwargs,
459 ) -> list[PromptReplacement]:
463 - num_image_tokens = self._get_num_image_tokens()
460 + num_image_tokens = self.info.get_num_image_tokens()
461
462 return [
463 PromptReplacement(
@@ -491,7 +488,9 @@ class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
488 return result
489
490
494 -@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
491 +@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor,
492 + info=Blip2ProcessingInfo,
493 + dummy_inputs=Blip2DummyInputsBuilder)
494 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
495
496 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/chameleon.py
+24 -23
@@ -30,10 +30,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
30 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31 MultiModalInputsV2, MultiModalKwargs,
32 NestedTensors, PlaceholderRange)
33 +from vllm.multimodal.parse import MultiModalDataItems
34 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 - MultiModalDataItems, ProcessingMixin,
35 - PromptReplacement)
36 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
35 + BaseProcessingInfo, PromptReplacement)
36 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 from vllm.sequence import IntermediateTensors
38 from vllm.utils import print_warning_once
39
@@ -49,33 +49,34 @@ class ChameleonImagePixelInputs(TypedDict):
49 """Shape: `(batch_size * num_images, num_channels, height, width)`"""
50
51
52 -class ChameleonProcessingMixin(ProcessingMixin):
52 +class ChameleonProcessingInfo(BaseProcessingInfo):
53
54 - def _get_hf_config(self):
54 + def get_hf_config(self):
55 return self.ctx.get_hf_config(ChameleonConfig)
56
57 - def _get_hf_processor(self):
57 + def get_hf_processor(self):
58 return self.ctx.get_hf_processor(ChameleonProcessor)
59
60 - def _get_num_image_tokens(self) -> int:
61 - processor = self._get_hf_processor()
62 - return processor.image_seq_length
63 -
64 -
65 -class ChameleonProfilingInfo(ChameleonProcessingMixin, BaseProfilingInfo):
66 -
60 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
61 return {"image": 1}
62
63 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
71 - return {"image": self._get_num_image_tokens()}
64 + return {"image": self.get_num_image_tokens()}
65 +
66 + def get_num_image_tokens(self) -> int:
67 + processor = self.get_hf_processor()
68 + return processor.image_seq_length
69 +
70 +
71 +class ChameleonDummyInputsBuilder(
72 + BaseDummyInputsBuilder[ChameleonProcessingInfo]):
73
74 def get_dummy_processor_inputs(
75 self,
76 seq_len: int,
77 mm_counts: Mapping[str, int],
78 ) -> ProcessorInputs:
78 - config = self._get_hf_config()
79 + config = self.info.get_hf_config()
80
81 width = height = config.vq_config.resolution
82 num_images = mm_counts.get("image", 0)
@@ -93,11 +94,8 @@ class ChameleonProfilingInfo(ChameleonProcessingMixin, BaseProfilingInfo):
94 )
95
96
96 -class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
97 - BaseMultiModalProcessor):
98 -
99 - def _get_profiling_info(self) -> BaseProfilingInfo:
100 - return ChameleonProfilingInfo(self.ctx)
97 +class ChameleonMultiModalProcessor(
98 + BaseMultiModalProcessor[ChameleonProcessingInfo]):
99
100 def _get_mm_fields_config(
101 self,
@@ -112,7 +110,7 @@ class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
110 hf_processor_mm_kwargs: Mapping[str, object],
111 out_mm_kwargs: MultiModalKwargs,
112 ) -> list[PromptReplacement]:
115 - processor = self._get_hf_processor(**hf_processor_mm_kwargs)
113 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
114
115 return [
116 PromptReplacement(
@@ -120,7 +118,7 @@ class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
118 target="<image>",
119 replacement="".join([
120 processor.image_start_token,
123 - processor.image_token * self._get_num_image_tokens(),
121 + processor.image_token * self.info.get_num_image_tokens(),
122 processor.image_end_token,
123 ]),
124 )
@@ -916,7 +914,10 @@ class ChameleonModel(nn.Module):
914 return hidden_states
915
916
919 -@MULTIMODAL_REGISTRY.register_processor(ChameleonMultiModalProcessor)
917 +@MULTIMODAL_REGISTRY.register_processor(
918 + ChameleonMultiModalProcessor,
919 + info=ChameleonProcessingInfo,
920 + dummy_inputs=ChameleonDummyInputsBuilder)
921 class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
922 SupportsPP):
923
vllm/model_executor/models/fuyu.py
+40 -40
@@ -33,11 +33,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 MultiModalInputsV2, MultiModalKwargs,
35 NestedTensors, PlaceholderRange)
36 -from vllm.multimodal.parse import ImageProcessorItems, ImageSize
36 +from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37 + MultiModalDataItems)
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
38 - MultiModalDataItems, ProcessingMixin,
39 - PromptReplacement)
40 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
39 + BaseProcessingInfo, PromptReplacement)
40 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
41 from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsMultiModal, SupportsPP
@@ -64,24 +64,38 @@ class FuyuImagePatchInputs(TypedDict):
64 """
65
66
67 -class FuyuProcessingMixin(ProcessingMixin):
67 +class FuyuProcessingInfo(BaseProcessingInfo):
68
69 - def _get_hf_config(self):
69 + def get_hf_config(self):
70 return self.ctx.get_hf_config(FuyuConfig)
71
72 - def _get_hf_processor(self):
72 + def get_hf_processor(self):
73 return self.ctx.get_hf_processor(FuyuProcessor)
74
75 - def _get_image_processor(self) -> FuyuImageProcessor:
76 - return self._get_hf_processor().image_processor
75 + def get_image_processor(self) -> FuyuImageProcessor:
76 + return self.get_hf_processor().image_processor
77
78 - def _get_image_feature_grid_size(
78 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
79 + return {"image": 1}
80 +
81 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
82 + target_width, target_height = self.get_image_size_with_most_features()
83 +
84 + max_ncols, max_nrows = self.get_image_feature_grid_size(
85 + image_width=target_width,
86 + image_height=target_height,
87 + )
88 + max_image_tokens = (max_ncols + 1) * max_nrows
89 +
90 + return {"image": max_image_tokens}
91 +
92 + def get_image_feature_grid_size(
93 self,
94 *,
95 image_width: int,
96 image_height: int,
97 ) -> tuple[int, int]:
84 - image_processor = self._get_image_processor()
98 + image_processor = self.get_image_processor()
99 target_width = image_processor.size["width"]
100 target_height = image_processor.size["height"]
101
@@ -97,34 +111,21 @@ class FuyuProcessingMixin(ProcessingMixin):
111 nrows = math.ceil(image_height / 30)
112 return ncols, nrows
113
100 -
101 -class FuyuProfilingInfo(FuyuProcessingMixin, BaseProfilingInfo):
102 -
103 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104 - return {"image": 1}
105 -
106 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
107 - target_width, target_height = self._get_image_size_with_most_features()
108 -
109 - max_ncols, max_nrows = self._get_image_feature_grid_size(
110 - image_width=target_width,
111 - image_height=target_height,
112 - )
113 - max_image_tokens = (max_ncols + 1) * max_nrows
114 -
115 - return {"image": max_image_tokens}
116 -
117 - def _get_image_size_with_most_features(self) -> ImageSize:
118 - image_processor = self._get_image_processor()
114 + def get_image_size_with_most_features(self) -> ImageSize:
115 + image_processor = self.get_image_processor()
116 return ImageSize(width=image_processor.size["width"],
117 height=image_processor.size["height"])
118
119 +
120 +class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
121 +
122 def get_dummy_processor_inputs(
123 self,
124 seq_len: int,
125 mm_counts: Mapping[str, int],
126 ) -> ProcessorInputs:
127 - target_width, target_height = self._get_image_size_with_most_features()
127 + target_width, target_height = \
128 + self.info.get_image_size_with_most_features()
129 num_images = mm_counts.get("image", 0)
130
131 mm_data = {
@@ -140,10 +141,7 @@ class FuyuProfilingInfo(FuyuProcessingMixin, BaseProfilingInfo):
141 )
142
143
143 -class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
144 -
145 - def _get_profiling_info(self) -> BaseProfilingInfo:
146 - return FuyuProfilingInfo(self.ctx)
144 +class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
145
146 def _call_hf_processor(
147 self,
@@ -156,7 +154,7 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
154 # Avoid warning from HF logger for text-only input
155 # Input_ids format: bos_token_id + prompt_token_ids + boa_token_id
156 # Tokenizer won't add boa_token_id by default, we add it manually.
159 - tokenizer = self._get_tokenizer()
157 + tokenizer = self.info.get_tokenizer()
158 boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
159 prompt_ids = tokenizer.encode(prompt) + [boa_token_id]
160 return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
@@ -196,10 +194,10 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
194 hf_processor_mm_kwargs: Mapping[str, object],
195 out_mm_kwargs: MultiModalKwargs,
196 ) -> list[PromptReplacement]:
199 - hf_config = self._get_hf_config()
197 + hf_config = self.info.get_hf_config()
198 bos_token_id = hf_config.bos_token_id
199
202 - tokenizer = self._get_tokenizer()
200 + tokenizer = self.info.get_tokenizer()
201 eot_token_id = tokenizer.bos_token_id
202 assert isinstance(eot_token_id, int)
203
@@ -207,7 +205,7 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
205 images = mm_items.get_items("image", ImageProcessorItems)
206 image_size = images.get_image_size(item_idx)
207
210 - ncols, nrows = self._get_image_feature_grid_size(
208 + ncols, nrows = self.info.get_image_feature_grid_size(
209 image_width=image_size.width,
210 image_height=image_size.height,
211 )
@@ -244,7 +242,9 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
242 return result
243
244
247 -@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
245 +@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor,
246 + info=FuyuProcessingInfo,
247 + dummy_inputs=FuyuDummyInputsBuilder)
248 class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
249
250 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/llava.py
+89 -86
@@ -1,7 +1,7 @@
1 -from abc import ABC, abstractmethod
1 +from abc import abstractmethod
2 from functools import cached_property
3 from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4 - Protocol, Set, Tuple, TypedDict, Union)
4 + Protocol, Set, Tuple, TypedDict, TypeVar, Union)
5
6 import torch
7 import torch.nn as nn
@@ -25,11 +25,11 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25 MultiModalInputsV2, MultiModalKwargs,
26 NestedTensors)
27 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28 - ImageSize)
28 + ImageSize, MultiModalDataItems)
29 from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 - MultiModalDataItems, ProcessingCache,
31 - ProcessingMixin, PromptReplacement)
32 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
30 + BaseProcessingInfo, ProcessingCache,
31 + PromptReplacement)
32 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
33 from vllm.sequence import IntermediateTensors
34
35 from .clip import CLIPVisionModel
@@ -105,34 +105,23 @@ class LlavaLikeProcessor(Protocol):
105 image_token: Final[str]
106
107
108 -class BaseLlavaProcessingMixin(ProcessingMixin, ABC):
108 +class BaseLlavaProcessingInfo(BaseProcessingInfo):
109
110 - def _get_hf_config(self) -> LlavaLikeConfig:
110 + def get_hf_config(self) -> LlavaLikeConfig:
111 return self.ctx.get_hf_config(LlavaConfig)
112
113 - def _get_vision_encoder_info(self):
114 - return get_vision_encoder_info(self._get_hf_config())
113 + def get_vision_encoder_info(self):
114 + return get_vision_encoder_info(self.get_hf_config())
115
116 @abstractmethod
117 - def _get_hf_processor(self) -> LlavaLikeProcessor:
117 + def get_hf_processor(self) -> LlavaLikeProcessor:
118 raise NotImplementedError
119
120 - def _get_num_image_tokens(
121 - self,
122 - *,
123 - image_width: int,
124 - image_height: int,
125 - ) -> int:
126 - hf_config = self._get_hf_config()
127 - vision_encoder_info = self._get_vision_encoder_info()
120 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
121 + return {"image": None}
122
129 - return self._apply_feature_select_strategy(
130 - hf_config.vision_feature_select_strategy,
131 - vision_encoder_info.get_num_image_tokens(
132 - image_width=image_width,
133 - image_height=image_height,
134 - ),
135 - )
123 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
124 + return {"image": self.get_max_image_tokens()}
125
126 def _apply_feature_select_strategy(
127 self,
@@ -147,28 +136,42 @@ class BaseLlavaProcessingMixin(ProcessingMixin, ABC):
136 msg = f"Unexpected feature select strategy: {strategy!r}"
137 raise NotImplementedError(msg)
138
139 + def get_num_image_tokens(
140 + self,
141 + *,
142 + image_width: int,
143 + image_height: int,
144 + ) -> int:
145 + hf_config = self.get_hf_config()
146 + vision_encoder_info = self.get_vision_encoder_info()
147
151 -class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
152 -
153 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
154 - return {"image": None}
155 -
156 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
157 - return {"image": self._get_max_image_tokens()}
148 + return self._apply_feature_select_strategy(
149 + hf_config.vision_feature_select_strategy,
150 + vision_encoder_info.get_num_image_tokens(
151 + image_width=image_width,
152 + image_height=image_height,
153 + ),
154 + )
155
159 - def _get_image_size_with_most_features(self) -> ImageSize:
160 - vision_encoder_info = self._get_vision_encoder_info()
156 + def get_image_size_with_most_features(self) -> ImageSize:
157 + vision_encoder_info = self.get_vision_encoder_info()
158 width = height = vision_encoder_info.get_image_size()
159 return ImageSize(width=width, height=height)
160
164 - def _get_max_image_tokens(self) -> int:
165 - target_width, target_height = self._get_image_size_with_most_features()
161 + def get_max_image_tokens(self) -> int:
162 + target_width, target_height = self.get_image_size_with_most_features()
163
167 - return self._get_num_image_tokens(
164 + return self.get_num_image_tokens(
165 image_width=target_width,
166 image_height=target_height,
167 )
168
169 +
170 +_I = TypeVar("_I", bound=BaseLlavaProcessingInfo)
171 +
172 +
173 +class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
174 +
175 def get_dummy_processor_inputs(
176 self,
177 seq_len: int,
@@ -176,9 +179,10 @@ class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
179 ) -> ProcessorInputs:
180 num_images = mm_counts.get("image", 0)
181
179 - processor = self._get_hf_processor()
182 + processor = self.info.get_hf_processor()
183 image_token = processor.image_token
181 - target_width, target_height = self._get_image_size_with_most_features()
184 + target_width, target_height = \
185 + self.info.get_image_size_with_most_features()
186
187 mm_data = {
188 "image":
@@ -193,23 +197,13 @@ class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
197 )
198
199
196 -class LlavaProcessingMixin(BaseLlavaProcessingMixin):
200 +class LlavaProcessingInfo(BaseLlavaProcessingInfo):
201
198 - def _get_hf_processor(self):
202 + def get_hf_processor(self):
203 return self.ctx.get_hf_processor(LlavaProcessor)
204
205
202 -class LlavaProfilingInfo(LlavaProcessingMixin, BaseLlavaProfilingInfo):
203 - pass
204 -
205 -
206 -class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
207 - BaseMultiModalProcessor):
208 -
209 - # Copied from BaseMultiModalProcessor
210 - @abstractmethod
211 - def _get_profiling_info(self) -> BaseProfilingInfo:
212 - raise NotImplementedError
206 +class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
207
208 # Copied from BaseMultiModalProcessor
209 @abstractmethod
@@ -226,7 +220,7 @@ class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
220 hf_processor_mm_kwargs: Mapping[str, object],
221 out_mm_kwargs: MultiModalKwargs,
222 ) -> list[PromptReplacement]:
229 - hf_config = self._get_hf_config()
223 + hf_config = self.info.get_hf_config()
224 image_token_id = hf_config.image_token_index
225
226 def get_replacement(item_idx: int):
@@ -237,7 +231,7 @@ class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
231 num_image_tokens = images.get_feature_size(item_idx)
232 else:
233 image_size = images.get_image_size(item_idx)
240 - num_image_tokens = self._get_num_image_tokens(
234 + num_image_tokens = self.info.get_num_image_tokens(
235 image_width=image_size.width,
236 image_height=image_size.height,
237 )
@@ -253,10 +247,8 @@ class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
247 ]
248
249
256 -class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
257 -
258 - def _get_profiling_info(self) -> BaseProfilingInfo:
259 - return LlavaProfilingInfo(self.ctx)
250 +class LlavaMultiModalProcessor(
251 + BaseLlavaMultiModalProcessor[LlavaProcessingInfo]):
252
253 def _get_mm_fields_config(
254 self,
@@ -269,21 +261,14 @@ class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
261 )
262
263
272 -class PixtralHFProcessingMixin(BaseLlavaProcessingMixin):
264 +class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
265
274 - def _get_hf_processor(self):
266 + def get_hf_processor(self):
267 return self.ctx.get_hf_processor(PixtralProcessor)
268
269
278 -class PixtralHFProfilingInfo(PixtralHFProcessingMixin, BaseLlavaProfilingInfo):
279 - pass
280 -
281 -
282 -class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
283 - BaseMultiModalProcessor):
284 -
285 - def _get_profiling_info(self) -> BaseProfilingInfo:
286 - return PixtralHFProfilingInfo(self.ctx)
270 +class PixtralHFMultiModalProcessor(
271 + BaseMultiModalProcessor[PixtralHFProcessingInfo]):
272
273 def _call_hf_processor(
274 self,
@@ -328,10 +313,10 @@ class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
313 hf_processor_mm_kwargs: Mapping[str, object],
314 out_mm_kwargs: MultiModalKwargs,
315 ) -> list[PromptReplacement]:
331 - hf_config = self._get_hf_config()
316 + hf_config = self.info.get_hf_config()
317 image_token_id = hf_config.image_token_index
318
334 - processor = self._get_hf_processor()
319 + processor = self.info.get_hf_processor()
320 image_token = processor.image_token
321 image_break_token = processor.image_break_token
322 image_end_token = processor.image_end_token
@@ -363,26 +348,40 @@ class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
348 ]
349
350
351 +def _build_llava_or_pixtral_hf_info(
352 + ctx: InputProcessingContext, ) -> BaseLlavaProcessingInfo:
353 + hf_config = ctx.get_hf_config(LlavaConfig)
354 +
355 + if isinstance(hf_config.vision_config, PixtralVisionConfig):
356 + return PixtralHFProcessingInfo(ctx)
357 +
358 + return LlavaProcessingInfo(ctx)
359 +
360 +
361 def _build_llava_or_pixtral_hf_processor(
367 - ctx: InputProcessingContext,
362 + info: _I,
363 + dummy_inputs: BaseDummyInputsBuilder[_I],
364 *,
365 cache: Optional[ProcessingCache] = None,
366 enable_sanity_checks: bool = True,
367 ) -> BaseMultiModalProcessor:
372 - hf_config = ctx.get_hf_config(LlavaConfig)
373 -
374 - if isinstance(hf_config.vision_config, PixtralVisionConfig):
368 + if isinstance(info, PixtralHFProcessingInfo):
369 return PixtralHFMultiModalProcessor(
376 - ctx,
370 + info,
371 + dummy_inputs, # type: ignore
372 + cache=cache,
373 + enable_sanity_checks=enable_sanity_checks,
374 + )
375 +
376 + if isinstance(info, LlavaProcessingInfo):
377 + return LlavaMultiModalProcessor(
378 + info,
379 + dummy_inputs, # type: ignore
380 cache=cache,
381 enable_sanity_checks=enable_sanity_checks,
382 )
383
381 - return LlavaMultiModalProcessor(
382 - ctx,
383 - cache=cache,
384 - enable_sanity_checks=enable_sanity_checks,
385 - )
384 + raise NotImplementedError(type(info))
385
386
387 def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
@@ -460,7 +459,9 @@ def init_vision_tower_for_llava(
459 raise NotImplementedError(msg)
460
461
463 -@MULTIMODAL_REGISTRY.register_processor(_build_llava_or_pixtral_hf_processor)
462 +@MULTIMODAL_REGISTRY.register_processor(_build_llava_or_pixtral_hf_processor,
463 + info=_build_llava_or_pixtral_hf_info,
464 + dummy_inputs=LlavaDummyInputsBuilder)
465 class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
466 # BitandBytes specific attributes
467 bitsandbytes_stacked_params_mapping = {
@@ -727,11 +728,11 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
728 mm_data: MultiModalDataDict,
729 hf_processor_mm_kwargs: Mapping[str, object],
730 ) -> MultiModalInputsV2:
730 - hf_config = self._get_hf_config()
731 + hf_config = self.info.get_hf_config()
732 image_token_id = hf_config.image_token_index
733
734 # Assume that it doesn't depend on the image size
734 - num_image_tokens = self._get_num_image_tokens(
735 + num_image_tokens = self.info.get_num_image_tokens(
736 image_width=-1,
737 image_height=-1,
738 )
@@ -796,6 +797,8 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
797
798 # To use this model, please use
799 # `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
799 -@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor)
800 +@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor,
801 + info=LlavaProcessingInfo,
802 + dummy_inputs=LlavaDummyInputsBuilder)
803 class MantisForConditionalGeneration(LlavaForConditionalGeneration):
804 pass
vllm/model_executor/models/llava_next.py
+33 -22
@@ -1,6 +1,7 @@
1 +from abc import abstractmethod
2 from functools import cached_property
3 from typing import (Final, Iterable, List, Literal, Mapping, Optional,
3 - Protocol, Set, Tuple, TypedDict, Union)
4 + Protocol, Set, Tuple, TypedDict, TypeVar, Union)
5
6 import torch
7 import torch.nn as nn
@@ -16,13 +17,12 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
19 from vllm.multimodal.parse import ImageSize
19 -from vllm.multimodal.profiling import BaseProfilingInfo
20 from vllm.sequence import IntermediateTensors
21
22 from .clip import CLIPVisionModel
23 from .interfaces import SupportsMultiModal, SupportsPP
24 -from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingMixin,
25 - BaseLlavaProfilingInfo, LlavaLikeConfig,
24 +from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingInfo,
25 + LlavaDummyInputsBuilder, LlavaLikeConfig,
26 LlavaMultiModalProjector, init_vision_tower_for_llava)
27 from .siglip import SiglipVisionModel
28 from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
@@ -65,23 +65,23 @@ class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
65 image_grid_pinpoints: Final[list[list[int]]]
66
67
68 -class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
68 +class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
69
70 - def _get_hf_config(self) -> LlavaNextLikeConfig:
70 + def get_hf_config(self) -> LlavaNextLikeConfig:
71 return self.ctx.get_hf_config(LlavaNextConfig)
72
73 - def _get_hf_processor(self):
73 + def get_hf_processor(self):
74 return self.ctx.get_hf_processor(LlavaNextProcessor)
75
76 # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
77 - def _get_num_image_tokens(
77 + def get_num_image_tokens(
78 self,
79 *,
80 image_width: int,
81 image_height: int,
82 ) -> int:
83 - hf_config = self._get_hf_config()
84 - vision_encoder_info = self._get_vision_encoder_info()
83 + hf_config = self.get_hf_config()
84 + vision_encoder_info = self.get_vision_encoder_info()
85
86 base_feature_size = self._apply_feature_select_strategy(
87 hf_config.vision_feature_select_strategy,
@@ -140,16 +140,13 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
140
141 return (unpadded_features, newline_features)
142
143 -
144 -class LlavaNextProfilingInfo(LlavaNextProcessingMixin, BaseLlavaProfilingInfo):
145 -
146 - def _get_image_size_with_most_features(self) -> ImageSize:
147 - hf_config = self._get_hf_config()
143 + def get_image_size_with_most_features(self) -> ImageSize:
144 + hf_config = self.get_hf_config()
145
146 largest_feature_size, largest_feature_pinpoint = 0, None
147 for (height, width) in hf_config.image_grid_pinpoints:
151 - feat_size = self._get_num_image_tokens(image_width=width,
152 - image_height=height)
148 + feat_size = self.get_num_image_tokens(image_width=width,
149 + image_height=height)
150 if feat_size > largest_feature_size:
151 largest_feature_size = feat_size
152 largest_feature_pinpoint = ImageSize(width=width,
@@ -161,11 +158,23 @@ class LlavaNextProfilingInfo(LlavaNextProcessingMixin, BaseLlavaProfilingInfo):
158 return largest_feature_pinpoint
159
160
164 -class LlavaNextMultiModalProcessor(LlavaNextProcessingMixin,
165 - BaseLlavaMultiModalProcessor):
161 +_I = TypeVar("_I", bound=LlavaNextProcessingInfo)
162 +
163 +
164 +class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
165 +
166 + # Copied from BaseMultiModalProcessor
167 + @abstractmethod
168 + def _get_mm_fields_config(
169 + self,
170 + hf_inputs: BatchFeature,
171 + hf_processor_mm_kwargs: Mapping[str, object],
172 + ) -> Mapping[str, MultiModalFieldConfig]:
173 + raise NotImplementedError
174 +
175
167 - def _get_profiling_info(self) -> BaseProfilingInfo:
168 - return LlavaNextProfilingInfo(self.ctx)
176 +class LlavaNextMultiModalProcessor(
177 + BaseLlavaNextMultiModalProcessor[LlavaNextProcessingInfo]):
178
179 def _get_mm_fields_config(
180 self,
@@ -179,7 +188,9 @@ class LlavaNextMultiModalProcessor(LlavaNextProcessingMixin,
188 )
189
190
182 -@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor)
191 +@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor,
192 + info=LlavaNextProcessingInfo,
193 + dummy_inputs=LlavaDummyInputsBuilder)
194 class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
195 SupportsPP):
196
vllm/model_executor/models/llava_next_video.py
+54 -50
@@ -17,12 +17,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
19 NestedTensors)
20 -from vllm.multimodal.parse import (ImageSize, VideoEmbeddingItems,
21 - VideoProcessorItems)
20 +from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
21 + VideoEmbeddingItems, VideoProcessorItems)
22 from vllm.multimodal.processing import (BaseMultiModalProcessor,
23 - MultiModalDataItems, ProcessingMixin,
24 - PromptReplacement)
25 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
23 + BaseProcessingInfo, PromptReplacement)
24 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25 from vllm.sequence import IntermediateTensors
26 from vllm.utils import is_list_of
27
@@ -47,33 +46,52 @@ class LlavaNextVideoPixelInputs(TypedDict):
46 """
47
48
50 -class LlavaNextVideoProcessingMixin(ProcessingMixin):
49 +class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
50
52 - def _get_hf_config(self):
51 + def get_hf_config(self):
52 return self.ctx.get_hf_config(LlavaNextVideoConfig)
53
55 - def _get_vision_encoder_info(self):
56 - return get_vision_encoder_info(self._get_hf_config())
54 + def get_vision_encoder_info(self):
55 + return get_vision_encoder_info(self.get_hf_config())
56
58 - def _get_hf_processor(self):
57 + def get_hf_processor(self):
58 return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
59
60 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
61 + return {"video": 1}
62 +
63 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
64 + target_width, target_height = self.get_image_size_with_most_features()
65 +
66 + max_video_tokens = self.get_num_video_tokens(
67 + image_width=target_width,
68 + image_height=target_height,
69 + num_frames=self.get_num_frames_with_most_features(seq_len),
70 + )
71 +
72 + return {"video": max_video_tokens}
73 +
74 + def get_image_size_with_most_features(self) -> ImageSize:
75 + vision_encoder_info = self.get_vision_encoder_info()
76 + width = height = vision_encoder_info.get_image_size()
77 + return ImageSize(width=width, height=height)
78 +
79 def _get_num_frame_tokens(
80 self,
81 *,
82 image_width: int,
83 image_height: int,
84 ) -> int:
67 - hf_config = self._get_hf_config()
85 + hf_config = self.get_hf_config()
86 spatial_pool_stride = hf_config.spatial_pool_stride
87
70 - vision_encoder_info = self._get_vision_encoder_info()
88 + vision_encoder_info = self.get_vision_encoder_info()
89 patch_grid_length = vision_encoder_info.get_patch_grid_length()
90 pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
91
92 return pooled_grid_length * pooled_grid_length
93
76 - def _get_num_video_tokens(
94 + def get_num_video_tokens(
95 self,
96 *,
97 image_width: int,
@@ -87,37 +105,14 @@ class LlavaNextVideoProcessingMixin(ProcessingMixin):
105
106 return num_frame_tokens * num_frames
107
90 -
91 -class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
92 - BaseProfilingInfo):
93 -
94 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
95 - return {"video": 1}
96 -
97 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
98 - target_width, target_height = self._get_image_size_with_most_features()
99 -
100 - max_video_tokens = self._get_num_video_tokens(
101 - image_width=target_width,
102 - image_height=target_height,
103 - num_frames=self._get_dummy_num_frames(seq_len),
104 - )
105 -
106 - return {"video": max_video_tokens}
107 -
108 - def _get_image_size_with_most_features(self) -> ImageSize:
109 - vision_encoder_info = self._get_vision_encoder_info()
110 - width = height = vision_encoder_info.get_image_size()
111 - return ImageSize(width=width, height=height)
112 -
108 def _get_max_video_frames(self, max_tokens: int) -> int:
114 - target_width, target_height = self._get_image_size_with_most_features()
109 + target_width, target_height = self.get_image_size_with_most_features()
110
111 num_frames = 0
112
113 while True:
114 next_num_frames = num_frames + 1
120 - next_max_tokens = self._get_num_video_tokens(
115 + next_max_tokens = self.get_num_video_tokens(
116 image_width=target_width,
117 image_height=target_height,
118 num_frames=next_num_frames,
@@ -130,7 +125,7 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
125
126 return num_frames
127
133 - def _get_dummy_num_frames(self, seq_len: int) -> int:
128 + def get_num_frames_with_most_features(self, seq_len: int) -> int:
129 mm_config = self.ctx.get_mm_config()
130 max_videos = mm_config.limit_per_prompt.get("video", 1)
131
@@ -138,6 +133,10 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
133
134 return max(max_total_frames // max(max_videos, 1), 1)
135
136 +
137 +class LlavaNextVideoDummyInputsBuilder(
138 + BaseDummyInputsBuilder[LlavaNextVideoProcessingInfo]):
139 +
140 def get_dummy_processor_inputs(
141 self,
142 seq_len: int,
@@ -145,16 +144,20 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
144 ) -> ProcessorInputs:
145 num_videos = mm_counts.get("video", 0)
146
148 - processor = self._get_hf_processor()
147 + processor = self.info.get_hf_processor()
148 video_token = processor.video_token
150 - target_width, target_height = self._get_image_size_with_most_features()
149 +
150 + target_width, target_height = \
151 + self.info.get_image_size_with_most_features()
152 + target_num_frames = \
153 + self.info.get_num_frames_with_most_features(seq_len)
154
155 mm_data = {
156 "video":
157 self._get_dummy_videos(
158 width=target_width,
159 height=target_height,
157 - num_frames=self._get_dummy_num_frames(seq_len),
160 + num_frames=target_num_frames,
161 num_videos=num_videos,
162 )
163 }
@@ -165,11 +168,8 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
168 )
169
170
168 -class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
169 - BaseMultiModalProcessor):
170 -
171 - def _get_profiling_info(self) -> BaseProfilingInfo:
172 - return LlavaNextVideoProfilingInfo(self.ctx)
171 +class LlavaNextVideoMultiModalProcessor(
172 + BaseMultiModalProcessor[LlavaNextVideoProcessingInfo]):
173
174 def _get_mm_fields_config(
175 self,
@@ -184,7 +184,7 @@ class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
184 hf_processor_mm_kwargs: Mapping[str, object],
185 out_mm_kwargs: MultiModalKwargs,
186 ) -> list[PromptReplacement]:
187 - hf_config = self._get_hf_config()
187 + hf_config = self.info.get_hf_config()
188 video_token_id = hf_config.video_token_index
189
190 def get_replacement(item_idx: int):
@@ -195,7 +195,7 @@ class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
195 num_video_tokens = videos.get_feature_size(item_idx)
196 else:
197 image_size = videos.get_frame_size(item_idx)
198 - num_video_tokens = self._get_num_video_tokens(
198 + num_video_tokens = self.info.get_num_video_tokens(
199 image_width=image_size.width,
200 image_height=image_size.height,
201 num_frames=videos.get_num_frames(item_idx),
@@ -269,7 +269,11 @@ class LlavaNextMultiModalProjector(nn.Module):
269 return hidden_states
270
271
272 -@MULTIMODAL_REGISTRY.register_processor(LlavaNextVideoMultiModalProcessor)
272 +@MULTIMODAL_REGISTRY.register_processor(
273 + LlavaNextVideoMultiModalProcessor,
274 + info=LlavaNextVideoProcessingInfo,
275 + dummy_inputs=LlavaNextVideoDummyInputsBuilder,
276 +)
277 class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
278 SupportsPP):
279
vllm/model_executor/models/llava_onevision.py
+52 -63
@@ -17,19 +17,20 @@ from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
20 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
21 -from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
22 - VideoEmbeddingItems, VideoProcessorItems)
23 -from vllm.multimodal.processing import MultiModalFieldConfig, PromptReplacement
24 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
20 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
21 + NestedTensors)
22 +from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
23 + VideoProcessorItems)
24 +from vllm.multimodal.processing import PromptReplacement
25 +from vllm.multimodal.profiling import ProcessorInputs
26 from vllm.sequence import IntermediateTensors
27 from vllm.utils import is_list_of
28
29 from .clip import CLIPVisionModel
30 from .interfaces import SupportsMultiModal, SupportsPP
30 -from .llava import BaseLlavaProfilingInfo, init_vision_tower_for_llava
31 -from .llava_next import (LlavaNextLikeConfig, LlavaNextMultiModalProcessor,
32 - LlavaNextProcessingMixin)
31 +from .llava import LlavaDummyInputsBuilder, init_vision_tower_for_llava
32 +from .llava_next import (BaseLlavaNextMultiModalProcessor, LlavaNextLikeConfig,
33 + LlavaNextProcessingInfo)
34 from .siglip import SiglipVisionModel
35 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36 maybe_prefix, merge_multimodal_embeddings)
@@ -89,14 +90,23 @@ class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
90 video_token_index: Final[int]
91
92
92 -class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
93 +class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
94
94 - def _get_hf_config(self) -> LlavaOnevisionLikeConfig:
95 + def get_hf_config(self) -> LlavaOnevisionLikeConfig:
96 return self.ctx.get_hf_config(LlavaOnevisionConfig)
97
97 - def _get_hf_processor(self):
98 + def get_hf_processor(self):
99 return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
100
101 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
102 + return {"image": None, "video": None}
103 +
104 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
105 + return {
106 + "image": self.get_max_image_tokens(),
107 + "video": self.get_max_video_tokens(seq_len),
108 + }
109 +
110 # Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
111 # with additional logic afterwards taken from LlavaOnevisionProcessor
112 def _get_num_unpadded_features(
@@ -141,16 +151,16 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
151 image_width: int,
152 image_height: int,
153 ) -> int:
144 - hf_config = self._get_hf_config()
154 + hf_config = self.get_hf_config()
155 spatial_pool_stride = getattr(hf_config, "spatial_pool_stride", 2)
156
147 - vision_encoder_info = self._get_vision_encoder_info()
157 + vision_encoder_info = self.get_vision_encoder_info()
158 patch_grid_length = vision_encoder_info.get_patch_grid_length()
159 pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
160
161 return pooled_grid_length * pooled_grid_length
162
153 - def _get_num_video_tokens(
163 + def get_num_video_tokens(
164 self,
165 *,
166 image_width: int,
@@ -164,43 +174,14 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
174
175 return num_frame_tokens * num_frames + 1 # Newline token
176
167 -
168 -class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
169 - BaseLlavaProfilingInfo):
170 -
171 - def _get_image_size_with_most_features(self) -> ImageSize:
172 - hf_config = self._get_hf_config()
173 - largest_feature_size, largest_feature_pinpoint = 0, None
174 - for (height, width) in hf_config.image_grid_pinpoints:
175 - feat_size = self._get_num_image_tokens(image_width=width,
176 - image_height=height)
177 - if feat_size > largest_feature_size:
178 - largest_feature_size = feat_size
179 - largest_feature_pinpoint = ImageSize(width=width,
180 - height=height)
181 -
182 - if largest_feature_size == 0 or largest_feature_pinpoint is None:
183 - raise ValueError("Cannot have a largest feature size of 0!")
184 -
185 - return largest_feature_pinpoint
186 -
187 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
188 - return {"image": None, "video": None}
189 -
190 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
191 - return {
192 - "image": self._get_max_image_tokens(),
193 - "video": self._get_max_video_tokens(seq_len),
194 - }
195 -
177 def _get_max_video_frames(self, max_tokens: int) -> int:
197 - target_width, target_height = self._get_image_size_with_most_features()
178 + target_width, target_height = self.get_image_size_with_most_features()
179
180 num_frames = 0
181
182 while True:
183 next_num_frames = num_frames + 1
203 - next_max_tokens = self._get_num_video_tokens(
184 + next_max_tokens = self.get_num_video_tokens(
185 image_width=target_width,
186 image_height=target_height,
187 num_frames=next_num_frames,
@@ -213,12 +194,12 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
194
195 return num_frames
196
216 - def _get_dummy_num_frames(self, seq_len: int) -> int:
197 + def get_num_frames_with_most_features(self, seq_len: int) -> int:
198 mm_config = self.ctx.get_mm_config()
199 max_images = mm_config.limit_per_prompt.get("image", 1)
200 max_videos = mm_config.limit_per_prompt.get("video", 1)
201
221 - max_image_tokens = self._get_max_image_tokens() * max_images
202 + max_image_tokens = self.get_max_image_tokens() * max_images
203 max_total_frames = self._get_max_video_frames(seq_len -
204 max_image_tokens)
205 max_frames_per_video = min(max_total_frames // max(max_videos, 1),
@@ -226,15 +207,19 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
207
208 return max(max_frames_per_video, 1)
209
229 - def _get_max_video_tokens(self, seq_len: int) -> int:
230 - target_width, target_height = self._get_image_size_with_most_features()
210 + def get_max_video_tokens(self, seq_len: int) -> int:
211 + target_width, target_height = self.get_image_size_with_most_features()
212
232 - return self._get_num_video_tokens(
213 + return self.get_num_video_tokens(
214 image_width=target_width,
215 image_height=target_height,
235 - num_frames=self._get_dummy_num_frames(seq_len),
216 + num_frames=self.get_num_frames_with_most_features(seq_len),
217 )
218
219 +
220 +class LlavaOnevisionDummyInputsBuilder(
221 + LlavaDummyInputsBuilder[LlavaOnevisionProcessingInfo]):
222 +
223 def get_dummy_processor_inputs(
224 self,
225 seq_len: int,
@@ -243,10 +228,14 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
228 num_images = mm_counts.get("image", 0)
229 num_videos = mm_counts.get("video", 0)
230
246 - processor = self._get_hf_processor()
231 + processor = self.info.get_hf_processor()
232 image_token = processor.image_token
233 video_token = processor.video_token
249 - target_width, target_height = self._get_image_size_with_most_features()
234 +
235 + target_width, target_height = \
236 + self.info.get_image_size_with_most_features()
237 + target_num_frames = \
238 + self.info.get_num_frames_with_most_features(seq_len)
239
240 mm_data = {
241 "image":
@@ -257,7 +246,7 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
246 self._get_dummy_videos(
247 width=target_width,
248 height=target_height,
260 - num_frames=self._get_dummy_num_frames(seq_len),
249 + num_frames=target_num_frames,
250 num_videos=num_videos,
251 )
252 }
@@ -268,11 +257,8 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
257 )
258
259
271 -class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
272 - LlavaNextMultiModalProcessor):
273 -
274 - def _get_profiling_info(self) -> BaseProfilingInfo:
275 - return LlavaOnevisionProfilingInfo(self.ctx)
260 +class LlavaOnevisionMultiModalProcessor(
261 + BaseLlavaNextMultiModalProcessor[LlavaOnevisionProcessingInfo]):
262
263 def _get_mm_fields_config(
264 self,
@@ -303,7 +289,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
289 mm_kwargs=mm_kwargs,
290 )
291
306 - processor = self._get_hf_processor()
292 + processor = self.info.get_hf_processor()
293 video_token = processor.video_token
294
295 # LLaVA-OneVision processor doesn't support multiple videos
@@ -345,7 +331,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
331 out_mm_kwargs=out_mm_kwargs,
332 )
333
348 - hf_config = self._get_hf_config()
334 + hf_config = self.info.get_hf_config()
335 video_token_id = hf_config.video_token_index
336
337 def get_video_replacement(item_idx: int):
@@ -356,7 +342,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
342 num_video_tokens = videos.get_feature_size(item_idx)
343 else:
344 image_size = videos.get_frame_size(item_idx)
359 - num_video_tokens = self._get_num_video_tokens(
345 + num_video_tokens = self.info.get_num_video_tokens(
346 image_width=image_size.width,
347 image_height=image_size.height,
348 num_frames=videos.get_num_frames(item_idx),
@@ -393,7 +379,10 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
379 return hidden_states
380
381
396 -@MULTIMODAL_REGISTRY.register_processor(LlavaOnevisionMultiModalProcessor)
382 +@MULTIMODAL_REGISTRY.register_processor(
383 + LlavaOnevisionMultiModalProcessor,
384 + info=LlavaOnevisionProcessingInfo,
385 + dummy_inputs=LlavaOnevisionDummyInputsBuilder)
386 class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
387 SupportsPP):
388
vllm/model_executor/models/phi3v.py
+43 -40
@@ -34,13 +34,12 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34 MultiModalInputsV2, MultiModalKwargs,
35 NestedTensors, PlaceholderRange)
36 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
37 - ImageSize)
37 + ImageSize, MultiModalDataItems)
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 - MultiModalDataItems, ProcessingMixin,
40 - PromptReplacement,
41 - _BoundPromptReplacement,
42 - _PlaceholderInfo)
43 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
39 + BaseProcessingInfo,
40 + BoundPromptReplacement,
41 + PlaceholderInfo, PromptReplacement)
42 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
43 from vllm.sequence import IntermediateTensors
44 from vllm.utils import is_list_of
45
@@ -302,9 +301,9 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
301 return image_features_hd_newline
302
303
305 -class Phi3VProcessingMixin(ProcessingMixin):
304 +class Phi3VProcessingInfo(BaseProcessingInfo):
305
307 - def _get_hf_processor(
306 + def get_hf_processor(
307 self,
308 *,
309 num_crops: Optional[int] = None,
@@ -314,39 +313,42 @@ class Phi3VProcessingMixin(ProcessingMixin):
313
314 return self.ctx.get_hf_processor()
315
317 - def _get_num_image_tokens(
318 - self,
319 - *,
320 - image_width: int,
321 - image_height: int,
322 - ) -> int:
323 - processor = self._get_hf_processor()
324 -
325 - return processor.calc_num_image_tokens_from_image_size( # type: ignore
326 - width=image_width,
327 - height=image_height,
328 - )
329 -
330 -
331 -class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
332 -
316 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
317 return {"image": None}
318
319 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
337 - target_width, target_height = self._get_image_size_with_most_features()
320 + target_width, target_height = self.get_image_size_with_most_features()
321
339 - max_image_tokens = self._get_num_image_tokens(
322 + max_image_tokens = self.get_num_image_tokens(
323 image_width=target_width,
324 image_height=target_height,
325 + processor=None,
326 )
327
328 return {"image": max_image_tokens}
329
346 - def _get_image_size_with_most_features(self) -> ImageSize:
330 + def get_num_image_tokens(
331 + self,
332 + *,
333 + image_width: int,
334 + image_height: int,
335 + processor: Optional[ProcessorMixin],
336 + ) -> int:
337 + if processor is None:
338 + processor = self.get_hf_processor()
339 +
340 + return processor.calc_num_image_tokens_from_image_size( # type: ignore
341 + width=image_width,
342 + height=image_height,
343 + )
344 +
345 + def get_image_size_with_most_features(self) -> ImageSize:
346 # Result in the max possible feature size (h:w = 16:1)
347 return ImageSize(height=8000, width=50)
348
349 +
350 +class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
351 +
352 def get_dummy_processor_inputs(
353 self,
354 seq_len: int,
@@ -354,7 +356,8 @@ class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
356 ) -> ProcessorInputs:
357 num_images = mm_counts.get("image", 0)
358
357 - target_width, target_height = self._get_image_size_with_most_features()
359 + target_width, target_height = \
360 + self.info.get_image_size_with_most_features()
361
362 mm_data = {
363 "image":
@@ -363,7 +366,7 @@ class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
366 num_images=num_images)
367 }
368
366 - hf_processor = self._get_hf_processor()
369 + hf_processor = self.info.get_hf_processor()
370 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
371
372 return ProcessorInputs(
@@ -372,10 +375,7 @@ class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
375 )
376
377
375 -class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
376 -
377 - def _get_profiling_info(self) -> BaseProfilingInfo:
378 - return Phi3VProfilingInfo(self.ctx)
378 +class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
379
380 def _call_hf_processor(
381 self,
@@ -416,10 +416,10 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
416 hf_processor_mm_kwargs: Mapping[str, Any],
417 out_mm_kwargs: MultiModalKwargs,
418 ) -> list[PromptReplacement]:
419 - hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
419 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
420 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
421
422 - tokenizer = self._get_tokenizer()
422 + tokenizer = self.info.get_tokenizer()
423 bos_token_id = tokenizer.bos_token_id
424 assert isinstance(bos_token_id, int)
425
@@ -431,9 +431,10 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
431 num_image_tokens = images.get_feature_size(item_idx)
432 else:
433 image_size = images.get_image_size(item_idx)
434 - num_image_tokens = self._get_num_image_tokens(
434 + num_image_tokens = self.info.get_num_image_tokens(
435 image_width=image_size.width,
436 image_height=image_size.height,
437 + processor=hf_processor,
438 )
439
440 return [_IMAGE_TOKEN_ID] * num_image_tokens + [bos_token_id]
@@ -451,9 +452,9 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
452 def _apply_prompt_replacements(
453 self,
454 token_ids: list[int],
454 - mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
455 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
456 mm_item_counts: Mapping[str, int],
456 - ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
457 + ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
458 token_ids, text, placeholders = super()._apply_prompt_replacements(
459 token_ids=token_ids,
460 mm_prompt_repls=mm_prompt_repls,
@@ -466,7 +467,7 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
467 token_ids = [token_ids[0], *token_ids[2:]]
468 placeholders = {
469 modality: [
469 - _PlaceholderInfo(
470 + PlaceholderInfo(
471 modality=p.modality,
472 item_idx=p.item_idx,
473 start_idx=p.start_idx - 1,
@@ -499,7 +500,9 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
500 return result
501
502
502 -@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
503 +@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor,
504 + info=Phi3VProcessingInfo,
505 + dummy_inputs=Phi3VDummyInputsBuilder)
506 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
507 hf_to_vllm_mapper = WeightsMapper(
508 orig_to_new_prefix={
vllm/model_executor/models/qwen2_audio.py
+25 -24
@@ -38,11 +38,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
39 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40 NestedTensors)
41 -from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser
41 +from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
42 + MultiModalDataParser)
43 from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 - MultiModalDataItems, ProcessingMixin,
44 - PromptReplacement)
45 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
44 + BaseProcessingInfo, PromptReplacement)
45 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46 from vllm.sequence import IntermediateTensors
47
48 from .interfaces import SupportsMultiModal, SupportsPP
@@ -80,12 +80,12 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
80 return feat_lengths, output_lengths
81
82
83 -class Qwen2AudioProcessingMixin(ProcessingMixin):
83 +class Qwen2AudioProcessingInfo(BaseProcessingInfo):
84
85 - def _get_hf_config(self):
85 + def get_hf_config(self):
86 return self.ctx.get_hf_config(Qwen2AudioConfig)
87
88 - def _get_hf_processor(
88 + def get_hf_processor(
89 self,
90 *,
91 # Ignored in initialization
@@ -93,36 +93,37 @@ class Qwen2AudioProcessingMixin(ProcessingMixin):
93 ) -> Qwen2AudioProcessor:
94 return self.ctx.get_hf_processor(Qwen2AudioProcessor)
95
96 - def _get_feature_extractor(
96 + def get_feature_extractor(
97 self,
98 *,
99 # Ignored in initialization
100 sampling_rate: Optional[int] = None,
101 ) -> WhisperFeatureExtractor:
102 - hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
102 + hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
103 feature_extractor = hf_processor.feature_extractor # type: ignore
104 assert isinstance(feature_extractor, WhisperFeatureExtractor)
105 return feature_extractor
106
107 -
108 -class Qwen2AudioProfilingInfo(Qwen2AudioProcessingMixin, BaseProfilingInfo):
109 -
107 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
108 return {"audio": None}
109
110 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
114 - hf_config = self._get_hf_config()
111 + hf_config = self.get_hf_config()
112 max_source_positions = hf_config.audio_config.max_source_positions
113 max_output_lengths = (max_source_positions - 2) // 2 + 1
114
115 return {"audio": max_output_lengths}
116
117 +
118 +class Qwen2AudioDummyInputsBuilder(
119 + BaseDummyInputsBuilder[Qwen2AudioProcessingInfo]):
120 +
121 def get_dummy_processor_inputs(
122 self,
123 seq_len: int,
124 mm_counts: Mapping[str, int],
125 ) -> ProcessorInputs:
125 - feature_extractor = self._get_feature_extractor()
126 + feature_extractor = self.info.get_feature_extractor()
127
128 sampling_rate = feature_extractor.sampling_rate
129 audio_len = feature_extractor.chunk_length * sampling_rate
@@ -139,14 +140,11 @@ class Qwen2AudioProfilingInfo(Qwen2AudioProcessingMixin, BaseProfilingInfo):
140 )
141
142
142 -class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
143 - BaseMultiModalProcessor):
144 -
145 - def _get_profiling_info(self) -> BaseProfilingInfo:
146 - return Qwen2AudioProfilingInfo(self.ctx)
143 +class Qwen2AudioMultiModalProcessor(
144 + BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
145
146 def _get_data_parser(self) -> MultiModalDataParser:
149 - feature_extractor = self._get_feature_extractor()
147 + feature_extractor = self.info.get_feature_extractor()
148 return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
149
150 def _call_hf_processor(
@@ -161,7 +159,7 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
159 if audios:
160 mm_data["audios"] = audios
161
164 - feature_extractor = self._get_feature_extractor(**mm_kwargs)
162 + feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
163 mm_kwargs = dict(
164 **mm_kwargs,
165 sampling_rate=feature_extractor.sampling_rate,
@@ -194,7 +192,7 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
192 hf_processor_mm_kwargs: Mapping[str, object],
193 out_mm_kwargs: MultiModalKwargs,
194 ) -> list[PromptReplacement]:
197 - hf_config = self._get_hf_config()
195 + hf_config = self.info.get_hf_config()
196 placeholder = hf_config.audio_token_index
197
198 feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
@@ -234,10 +232,13 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
232 # has already performed processing for multi-audio input when the input
233 # audios are short (the corresponding placeholders may take up fewer
234 # tokens than the number of audio items)
237 - return not hasattr(self._get_hf_processor(), "audio_token")
235 + return not hasattr(self.info.get_hf_processor(), "audio_token")
236
237
240 -@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
238 +@MULTIMODAL_REGISTRY.register_processor(
239 + Qwen2AudioMultiModalProcessor,
240 + info=Qwen2AudioProcessingInfo,
241 + dummy_inputs=Qwen2AudioDummyInputsBuilder)
242 class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
243 SupportsPP):
244
vllm/model_executor/models/qwen2_vl.py
+63 -50
@@ -57,11 +57,10 @@ from vllm.multimodal.inputs import (ImageItem, ModalityData,
57 MultiModalFieldConfig, MultiModalKwargs,
58 NestedTensors, VideoItem)
59 from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60 - MultiModalDataParser)
60 + MultiModalDataItems, MultiModalDataParser)
61 from vllm.multimodal.processing import (BaseMultiModalProcessor,
62 - MultiModalDataItems, ProcessingMixin,
63 - PromptReplacement)
64 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
62 + BaseProcessingInfo, PromptReplacement)
63 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
64 from vllm.platforms import _Backend
65 from vllm.sequence import IntermediateTensors
66 from vllm.transformers_utils.config import uses_mrope
@@ -709,12 +708,12 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
708 return super()._parse_video_data(data)
709
710
712 -class Qwen2VLProcessingMixin(ProcessingMixin):
711 +class Qwen2VLProcessingInfo(BaseProcessingInfo):
712
714 - def _get_hf_config(self):
713 + def get_hf_config(self):
714 return self.ctx.get_hf_config(Qwen2VLConfig)
715
717 - def _get_hf_processor(
716 + def get_hf_processor(
717 self,
718 *,
719 min_pixels: Optional[int] = None,
@@ -736,18 +735,27 @@ class Qwen2VLProcessingMixin(ProcessingMixin):
735
736 return hf_processor
737
739 - def _get_image_processor(
738 + def get_image_processor(
739 self,
740 *,
741 min_pixels: Optional[int] = None,
742 max_pixels: Optional[int] = None,
743 ):
745 - hf_processor = self._get_hf_processor(min_pixels=min_pixels,
746 - max_pixels=max_pixels)
744 + hf_processor = self.get_hf_processor(min_pixels=min_pixels,
745 + max_pixels=max_pixels)
746 image_processor = hf_processor.image_processor # type: ignore
747 assert isinstance(image_processor, Qwen2VLImageProcessor)
748 return image_processor
749
750 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
751 + return {"image": None, "video": None}
752 +
753 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
754 + return {
755 + "image": self.get_max_image_tokens(),
756 + "video": self.get_max_video_tokens(seq_len),
757 + }
758 +
759 def _get_vision_info(
760 self,
761 *,
@@ -755,15 +763,17 @@ class Qwen2VLProcessingMixin(ProcessingMixin):
763 image_height: int,
764 num_frames: int = 1,
765 do_resize: bool = True,
766 + image_processor: Optional[Qwen2VLImageProcessor],
767 ) -> tuple[ImageSize, int]:
759 - hf_config = self._get_hf_config()
768 + if image_processor is None:
769 + image_processor = self.get_image_processor()
770 +
771 + hf_config = self.get_hf_config()
772 vision_config = hf_config.vision_config
773 patch_size = vision_config.patch_size
774 merge_size = vision_config.spatial_merge_size
775 temporal_patch_size = vision_config.temporal_patch_size
776
765 - image_processor = self._get_image_processor()
766 -
777 if do_resize:
778 resized_height, resized_width = smart_resize(
779 height=image_height,
@@ -787,70 +797,65 @@ class Qwen2VLProcessingMixin(ProcessingMixin):
797
798 return preprocessed_size, num_vision_tokens
799
790 - def _get_num_image_tokens(
800 + def get_num_image_tokens(
801 self,
802 *,
803 image_width: int,
804 image_height: int,
805 + image_processor: Optional[Qwen2VLImageProcessor],
806 ) -> int:
807 _, num_image_tokens = self._get_vision_info(
808 image_width=image_width,
809 image_height=image_height,
810 + image_processor=image_processor,
811 )
812 return num_image_tokens
813
802 - def _get_num_video_tokens(
814 + def get_num_video_tokens(
815 self,
816 *,
817 image_width: int,
818 image_height: int,
819 num_frames: int,
820 + image_processor: Optional[Qwen2VLImageProcessor],
821 ) -> int:
822 _, num_video_tokens = self._get_vision_info(
823 image_width=image_width,
824 image_height=image_height,
825 num_frames=num_frames,
826 + image_processor=image_processor,
827 )
828 return num_video_tokens
829
816 -
817 -class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
818 -
819 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
820 - return {"image": None, "video": None}
821 -
822 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
823 - return {
824 - "image": self._get_max_image_tokens(),
825 - "video": self._get_max_video_tokens(seq_len),
826 - }
827 -
828 - def _get_image_size_with_most_features(self) -> ImageSize:
830 + def get_image_size_with_most_features(self) -> ImageSize:
831 max_image_size, _ = self._get_vision_info(
832 image_width=9999999,
833 image_height=9999999,
834 + image_processor=None,
835 )
836 return max_image_size
837
835 - def _get_max_image_tokens(self) -> int:
836 - target_width, target_height = self._get_image_size_with_most_features()
838 + def get_max_image_tokens(self) -> int:
839 + target_width, target_height = self.get_image_size_with_most_features()
840
838 - return self._get_num_image_tokens(
841 + return self.get_num_image_tokens(
842 image_width=target_width,
843 image_height=target_height,
844 + image_processor=None,
845 )
846
847 def _get_max_video_frames(self, max_tokens: int) -> int:
844 - target_width, target_height = self._get_image_size_with_most_features()
848 + target_width, target_height = self.get_image_size_with_most_features()
849
850 num_frames = 0
851
852 while True:
853 next_num_frames = num_frames + 1
850 - next_max_tokens = self._get_num_video_tokens(
854 + next_max_tokens = self.get_num_video_tokens(
855 image_width=target_width,
856 image_height=target_height,
857 num_frames=next_num_frames,
858 + image_processor=None,
859 )
860
861 if next_max_tokens > max_tokens:
@@ -860,12 +865,12 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
865
866 return num_frames
867
863 - def _get_dummy_num_frames(self, seq_len: int) -> int:
868 + def get_num_frames_with_most_features(self, seq_len: int) -> int:
869 mm_config = self.ctx.get_mm_config()
870 max_images = mm_config.limit_per_prompt.get("image", 1)
871 max_videos = mm_config.limit_per_prompt.get("video", 1)
872
868 - max_image_tokens = self._get_max_image_tokens() * max_images
873 + max_image_tokens = self.get_max_image_tokens() * max_images
874 max_total_frames = self._get_max_video_frames(seq_len -
875 max_image_tokens)
876
@@ -877,15 +882,19 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
882
883 return num_frames
884
880 - def _get_max_video_tokens(self, seq_len: int) -> int:
881 - target_width, target_height = self._get_image_size_with_most_features()
885 + def get_max_video_tokens(self, seq_len: int) -> int:
886 + target_width, target_height = self.get_image_size_with_most_features()
887
883 - return self._get_num_video_tokens(
888 + return self.get_num_video_tokens(
889 image_width=target_width,
890 image_height=target_height,
886 - num_frames=self._get_dummy_num_frames(seq_len),
891 + num_frames=self.get_num_frames_with_most_features(seq_len),
892 + image_processor=None,
893 )
894
895 +
896 +class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
897 +
898 def get_dummy_processor_inputs(
899 self,
900 seq_len: int,
@@ -894,10 +903,14 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
903 num_images = mm_counts.get("image", 0)
904 num_videos = mm_counts.get("video", 0)
905
897 - hf_processor = self._get_hf_processor()
906 + hf_processor = self.info.get_hf_processor()
907 image_token: str = hf_processor.image_token
908 video_token: str = hf_processor.video_token
900 - target_width, target_height = self._get_image_size_with_most_features()
909 +
910 + target_width, target_height = \
911 + self.info.get_image_size_with_most_features()
912 + target_num_frames = \
913 + self.info.get_num_frames_with_most_features(seq_len)
914
915 mm_data = {
916 "image":
@@ -908,7 +921,7 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
921 self._get_dummy_videos(
922 width=target_width,
923 height=target_height,
911 - num_frames=self._get_dummy_num_frames(seq_len),
924 + num_frames=target_num_frames,
925 num_videos=num_videos,
926 )
927 }
@@ -919,11 +932,8 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
932 )
933
934
922 -class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
923 - BaseMultiModalProcessor):
924 -
925 - def _get_profiling_info(self) -> BaseProfilingInfo:
926 - return Qwen2VLProfilingInfo(self.ctx)
935 +class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
936 + ):
937
938 def _get_data_parser(self) -> MultiModalDataParser:
939 return Qwen2MultiModalDataParser()
@@ -934,8 +944,9 @@ class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
944 hf_processor_mm_kwargs: Mapping[str, Any],
945 out_mm_kwargs: MultiModalKwargs,
946 ) -> list[PromptReplacement]:
937 - hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
938 - image_processor = self._get_image_processor(**hf_processor_mm_kwargs)
947 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
948 + image_processor = self.info.get_image_processor(
949 + **hf_processor_mm_kwargs)
950
951 # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
952 # image_token and video_token registered
@@ -991,7 +1002,9 @@ class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
1002 )
1003
1004
994 -@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
1005 +@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor,
1006 + info=Qwen2VLProcessingInfo,
1007 + dummy_inputs=Qwen2VLDummyInputsBuilder)
1008 class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1009 SupportsLoRA, SupportsPP):
1010 packed_modules_mapping = {
vllm/model_executor/models/ultravox.py
+23 -23
@@ -24,11 +24,10 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
26 NestedTensors)
27 -from vllm.multimodal.parse import MultiModalDataParser
27 +from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
28 from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 - MultiModalDataItems, ProcessingMixin,
30 - PromptReplacement)
31 -from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
29 + BaseProcessingInfo, PromptReplacement)
30 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 from vllm.sequence import IntermediateTensors
32 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
33
@@ -59,9 +58,9 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
58 UltravoxAudioEmbeddingInputs]
59
60
62 -class UltravoxProcessingMixin(ProcessingMixin):
61 +class UltravoxProcessingInfo(BaseProcessingInfo):
62
64 - def _get_hf_processor(
63 + def get_hf_processor(
64 self,
65 *,
66 # Ignored in initialization
@@ -76,37 +75,38 @@ class UltravoxProcessingMixin(ProcessingMixin):
75 hf_processor.audio_token_replacement = _AUDIO_PLACEHOLDER_OVERRIDE
76 return hf_processor
77
79 - def _get_feature_extractor(
78 + def get_feature_extractor(
79 self,
80 *,
81 # Ignored in initialization
82 sampling_rate: Optional[int] = None,
83 ) -> WhisperFeatureExtractor:
85 - hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
84 + hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
85 audio_processor = hf_processor.audio_processor # type: ignore
86 feature_extractor = audio_processor.feature_extractor # type: ignore
87 assert isinstance(feature_extractor, WhisperFeatureExtractor)
88 return feature_extractor
89
91 -
92 -class UltravoxProfilingInfo(UltravoxProcessingMixin, BaseProfilingInfo):
93 -
90 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
91 return {"audio": None}
92
93 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
98 - feature_extractor = self._get_feature_extractor()
94 + feature_extractor = self.get_feature_extractor()
95 max_audio_tokens = math.ceil(feature_extractor.chunk_length *
96 _AUDIO_TOKENS_PER_SECOND)
97
98 return {"audio": max_audio_tokens}
99
100 +
101 +class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
102 + ):
103 +
104 def get_dummy_processor_inputs(
105 self,
106 seq_len: int,
107 mm_counts: Mapping[str, int],
108 ) -> ProcessorInputs:
109 - feature_extractor = self._get_feature_extractor()
109 + feature_extractor = self.info.get_feature_extractor()
110
111 sampling_rate = feature_extractor.sampling_rate
112 audio_len = feature_extractor.chunk_length * sampling_rate
@@ -123,14 +123,11 @@ class UltravoxProfilingInfo(UltravoxProcessingMixin, BaseProfilingInfo):
123 )
124
125
126 -class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
127 - BaseMultiModalProcessor):
128 -
129 - def _get_profiling_info(self) -> BaseProfilingInfo:
130 - return UltravoxProfilingInfo(self.ctx)
126 +class UltravoxMultiModalProcessor(
127 + BaseMultiModalProcessor[UltravoxProcessingInfo]):
128
129 def _get_data_parser(self) -> MultiModalDataParser:
133 - feature_extractor = self._get_feature_extractor()
130 + feature_extractor = self.info.get_feature_extractor()
131 return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
132
133 def _call_hf_processor(
@@ -141,7 +138,7 @@ class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
138 ) -> BatchFeature:
139 # Text-only input not supported in composite processor
140 if not mm_data:
144 - tokenizer = self._get_tokenizer()
141 + tokenizer = self.info.get_tokenizer()
142
143 prompt_ids = tokenizer.encode(
144 prompt,
@@ -160,7 +157,7 @@ class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
157 mm_kwargs=mm_kwargs,
158 )
159
163 - feature_extractor = self._get_feature_extractor()
160 + feature_extractor = self.info.get_feature_extractor()
161 mm_kwargs = dict(
162 **mm_kwargs,
163 sampling_rate=feature_extractor.sampling_rate,
@@ -208,7 +205,7 @@ class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
205 hf_processor_mm_kwargs: Mapping[str, Any],
206 out_mm_kwargs: MultiModalKwargs,
207 ) -> list[PromptReplacement]:
211 - hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
208 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
209 placeholder = hf_processor.audio_token_replacement # type: ignore
210
211 def get_replacement_ultravox(item_idx: int):
@@ -342,7 +339,10 @@ class ModifiedWhisperEncoder(WhisperEncoder):
339 return hidden_states
340
341
345 -@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor)
342 +@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor,
343 + info=UltravoxProcessingInfo,
344 + dummy_inputs=UltravoxDummyInputsBuilder
345 + )
346 class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
347
348 hf_to_vllm_mapper = WeightsMapper(
vllm/multimodal/processing.py
+85 -134
@@ -4,12 +4,13 @@ from collections import defaultdict
4 from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
5 from dataclasses import dataclass, field
6 from functools import lru_cache
7 -from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
7 +from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
8 + TypeVar, Union)
9
10 from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
11
11 -from vllm import envs
12 -from vllm.inputs import DummyData, InputProcessingContext
12 +import vllm.envs as envs
13 +from vllm.inputs import InputProcessingContext
14 from vllm.logger import init_logger
15 from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
16 encode_tokens)
@@ -20,7 +21,9 @@ from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
21 MultiModalInputsV2, MultiModalKwargs,
22 MultiModalKwargsItem, PlaceholderRange)
23 from .parse import MultiModalDataItems, MultiModalDataParser
23 -from .profiling import BaseProfilingInfo
24 +
25 +if TYPE_CHECKING:
26 + from .profiling import BaseDummyInputsBuilder
27
28 logger = init_logger(__name__)
29
@@ -46,8 +49,8 @@ class PromptReplacement:
49 if it does not depend on the input.
50 """
51
49 - def bind(self, tokenizer: AnyTokenizer) -> "_BoundPromptReplacement":
50 - return _BoundPromptReplacement(
52 + def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptReplacement":
53 + return BoundPromptReplacement(
54 tokenizer=tokenizer,
55 modality=self.modality,
56 _target=self.target,
@@ -128,7 +131,7 @@ class _BoundPromptSequence:
131
132
133 @dataclass
131 -class _BoundPromptReplacement:
134 +class BoundPromptReplacement:
135 tokenizer: AnyTokenizer = field(repr=False)
136 modality: str
137
@@ -207,7 +210,7 @@ def iter_token_matches(
210
211 @dataclass(repr=False)
212 class _PromptReplacementMatch(ABC):
210 - prompt_repl: _BoundPromptReplacement
213 + prompt_repl: BoundPromptReplacement
214
215 @property
216 def modality(self) -> str:
@@ -255,7 +258,7 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch):
258
259
260 @dataclass
258 -class _PlaceholderInfo:
261 +class PlaceholderInfo:
262 modality: str
263 item_idx: int
264 start_idx: int
@@ -274,7 +277,7 @@ class _PlaceholderInfo:
277
278 def find_token_matches(
279 prompt: list[int],
277 - prompt_repls: Sequence[_BoundPromptReplacement],
280 + prompt_repls: Sequence[BoundPromptReplacement],
281 ) -> list[_PromptReplacementTokenMatch]:
282 """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
283 return [
@@ -286,7 +289,7 @@ def find_token_matches(
289
290 def find_text_matches(
291 prompt: str,
289 - prompt_repls: Sequence[_BoundPromptReplacement],
292 + prompt_repls: Sequence[BoundPromptReplacement],
293 ) -> list[_PromptReplacementTextMatch]:
294 """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
295 return [
@@ -390,9 +393,9 @@ def replace_text_matches(
393 def _iter_modality_placeholders(
394 prompt: list[int],
395 modality: str,
393 - modality_repls: Sequence[_BoundPromptReplacement],
396 + modality_repls: Sequence[BoundPromptReplacement],
397 modal_item_count: int,
395 -) -> Iterable[_PlaceholderInfo]:
398 +) -> Iterable[PlaceholderInfo]:
399 if modal_item_count == 0:
400 return
401
@@ -413,7 +416,7 @@ def _iter_modality_placeholders(
416 continue
417
418 if prompt[start_idx:end_idx] == repl_tokens:
416 - yield _PlaceholderInfo(
419 + yield PlaceholderInfo(
420 modality=modality,
421 item_idx=item_idx,
422 start_idx=start_idx,
@@ -434,10 +437,10 @@ def _iter_modality_placeholders(
437
438
439 def _iter_placeholders(
437 - mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
440 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
441 prompt: list[int],
442 mm_item_counts: Mapping[str, int],
440 -) -> Iterable[_PlaceholderInfo]:
443 +) -> Iterable[PlaceholderInfo]:
444 """
445 For each modality, yield each set of placeholder tokens found in
446 :code:`prompt`.
@@ -455,10 +458,10 @@ def _iter_placeholders(
458
459
460 def find_mm_placeholders(
458 - mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
461 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
462 prompt: list[int],
463 mm_item_counts: Mapping[str, int],
461 -) -> Mapping[str, list[_PlaceholderInfo]]:
464 +) -> Mapping[str, list[PlaceholderInfo]]:
465 it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
466 return dict(full_groupby_modality(it))
467
@@ -524,29 +527,59 @@ class ProcessingCache:
527 self._cache.put(cache_key, output_kwargs)
528
529
527 -class ProcessingMixin:
528 - """
529 - Contains helper functions to perform processing.
530 +class BaseProcessingInfo:
531 + """Base class containing information to perform processing."""
532
531 - Not to be confused with :class:`transformers.ProcessorMixin`.
532 - """
533 - ctx: InputProcessingContext
533 + def __init__(self, ctx: InputProcessingContext) -> None:
534 + super().__init__()
535
535 - def _get_tokenizer(self) -> AnyTokenizer:
536 + self.ctx = ctx
537 +
538 + @property
539 + def model_id(self) -> str:
540 + return self.ctx.model_config.model
541 +
542 + def get_tokenizer(self) -> AnyTokenizer:
543 return self.ctx.tokenizer
544
538 - def _get_hf_config(self) -> PretrainedConfig:
545 + def get_hf_config(self) -> PretrainedConfig:
546 return self.ctx.get_hf_config()
547
541 - def _get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
548 + def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
549 """
550 Subclasses can override this method to handle
551 specific kwargs from model config or user inputs.
552 """
553 return self.ctx.get_hf_processor(**kwargs)
554
555 + @abstractmethod
556 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
557 + """
558 + Return the maximum supported number of items for each modality.
559 +
560 + A value of `None` means unlimited number of items.
561 +
562 + Omitting a modality from the returned dictionary means that
563 + it is not supported at all.
564 + """
565 + raise NotImplementedError
566 +
567 + @abstractmethod
568 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
569 + """
570 + Get the maximum possible number of tokens per data item
571 + for each modality.
572 +
573 + The dictionary returned by this method should have the same
574 + keys as that returned by :meth:`get_supported_mm_limits`.
575 + """
576 + raise NotImplementedError
577 +
578 +
579 +_I = TypeVar("_I", bound=BaseProcessingInfo)
580
549 -class BaseMultiModalProcessor(ProcessingMixin, ABC):
581 +
582 +class BaseMultiModalProcessor(ABC, Generic[_I]):
583 """
584 Abstract base class to process multi-modal inputs to be used in vLLM.
585
@@ -554,18 +587,19 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
587 """
588
589 def __init__(self,
557 - ctx: InputProcessingContext,
590 + info: _I,
591 + dummy_inputs: "BaseDummyInputsBuilder[_I]",
592 *,
593 cache: Optional[ProcessingCache] = None,
594 enable_sanity_checks: bool = True) -> None:
595 super().__init__()
596
563 - self.ctx = ctx
597 + self.info = info
598 + self.dummy_inputs = dummy_inputs
599 self.cache = cache
600 self.enable_sanity_checks = enable_sanity_checks
601
602 self.data_parser = self._get_data_parser()
568 - self.profiling_info = self._get_profiling_info()
603
604 def __call__(
605 self,
@@ -585,13 +619,6 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
619 """
620 return MultiModalDataParser()
621
588 - def _get_profiling_info(self) -> BaseProfilingInfo:
589 - """
590 - Get the profiling information to find the worst-case memory usage of
591 - the model.
592 - """
593 - raise NotImplementedError
594 -
622 def _to_mm_items(
623 self,
624 mm_data: MultiModalDataDict,
@@ -602,7 +629,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
629 """
630 mm_items = self.data_parser.parse_mm_data(mm_data)
631
605 - mm_limits = self.ctx.get_mm_config().limit_per_prompt
632 + mm_limits = self.info.ctx.get_mm_config().limit_per_prompt
633 for modality, items in mm_items.items():
634 limit = mm_limits.get(modality, 1)
635 if len(items) > limit:
@@ -646,19 +673,19 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
673
674 def _find_mm_placeholders(
675 self,
649 - mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
676 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
677 new_token_ids: list[int],
678 mm_item_counts: Mapping[str, int],
652 - ) -> Mapping[str, list[_PlaceholderInfo]]:
679 + ) -> Mapping[str, list[PlaceholderInfo]]:
680 return find_mm_placeholders(mm_prompt_repls, new_token_ids,
681 mm_item_counts)
682
683 def _get_hf_mm_data(
684 self,
685 mm_items: MultiModalDataItems,
659 - ) -> tuple[dict[str, Any], dict[str, Any]]:
660 - processor_data = dict[str, Any]()
661 - passthrough_data = dict[str, Any]()
686 + ) -> tuple[Mapping[str, object], Mapping[str, object]]:
687 + processor_data = dict[str, object]()
688 + passthrough_data = dict[str, object]()
689
690 for items in mm_items.values():
691 processor_data.update(items.get_processor_data())
@@ -678,8 +705,8 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
705 Call the HF processor on the prompt text and
706 associated multi-modal data.
707 """
681 - return self.ctx.call_hf_processor(
682 - self._get_hf_processor(**mm_kwargs),
708 + return self.info.ctx.call_hf_processor(
709 + self.info.get_hf_processor(**mm_kwargs),
710 dict(text=prompt, **mm_data),
711 mm_kwargs,
712 )
@@ -738,8 +765,8 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
765
766 # Some HF processors (e.g. Qwen2-VL) expect corresponding
767 # multi-modal tokens to be in the prompt text
741 - dummy_inputs = self.profiling_info.get_dummy_processor_inputs(
742 - self.ctx.model_config.max_model_len,
768 + dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
769 + self.info.ctx.model_config.max_model_len,
770 mm_missing_counts,
771 )
772
@@ -762,7 +789,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
789 caching the results and reusing cached results.
790 """
791 cache = self.cache
765 - model_id = self.ctx.model_config.model
792 + model_id = self.info.model_id
793
794 _, passthrough_data = self._get_hf_mm_data(mm_data_items)
795 if cache is None or passthrough_data:
@@ -838,8 +865,8 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
865 def _bind_and_group_repls(
866 self,
867 prompt_repls: list[PromptReplacement],
841 - ) -> dict[str, list[_BoundPromptReplacement]]:
842 - tokenizer = self._get_tokenizer()
868 + ) -> dict[str, list[BoundPromptReplacement]]:
869 + tokenizer = self.info.get_tokenizer()
870
871 it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
872 return dict(full_groupby_modality(it))
@@ -859,10 +886,10 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
886 def _apply_prompt_replacements(
887 self,
888 token_ids: list[int],
862 - mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
889 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
890 mm_item_counts: Mapping[str, int],
864 - ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
865 - tokenizer = self._get_tokenizer()
891 + ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
892 + tokenizer = self.info.get_tokenizer()
893
894 mm_token_matches = {
895 modality: find_token_matches(token_ids, prompt_repls)
@@ -950,7 +977,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
977
978 def _validate_mm_placeholders(
979 self,
953 - mm_placeholders: Mapping[str, list[_PlaceholderInfo]],
980 + mm_placeholders: Mapping[str, list[PlaceholderInfo]],
981 mm_item_counts: Mapping[str, int],
982 *,
983 allow_missing: bool = False,
@@ -1001,7 +1028,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1028 # instead of rehashing.
1029
1030 if envs.VLLM_USE_V1:
1004 - model_id = self.ctx.model_config.model
1031 + model_id = self.info.model_id
1032 mm_hashes = {
1033 modality: [
1034 MultiModalHasher.hash_kwargs(model_id=model_id,
@@ -1046,7 +1073,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1073 allow_missing=True,
1074 )
1075
1049 - mm_missing_repls = dict[str, list[_BoundPromptReplacement]]()
1076 + mm_missing_repls = dict[str, list[BoundPromptReplacement]]()
1077 for modality, missing_repl_count in mm_missing_repl_counts.items():
1078 if missing_repl_count == 0:
1079 mm_missing_repls[modality] = []
@@ -1059,7 +1086,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1086 # If HF processor already inserts placeholder tokens,
1087 # there is no need for us to insert them
1088 if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1062 - tokenizer = self._get_tokenizer()
1089 + tokenizer = self.info.get_tokenizer()
1090 prompt_text = decode_tokens(tokenizer, prompt_ids)
1091 mm_placeholders = hf_mm_placeholders
1092 else:
@@ -1090,79 +1117,3 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1117 mm_hashes=mm_hashes,
1118 mm_placeholders=mm_placeholder_ranges,
1119 )
1093 -
1094 - def _get_dummy_mm_inputs(
1095 - self,
1096 - seq_len: int,
1097 - mm_counts: Mapping[str, int],
1098 - ) -> MultiModalInputsV2:
1099 - profiling = self.profiling_info
1100 - processor_inputs = profiling.get_dummy_processor_inputs(
1101 - seq_len, mm_counts)
1102 -
1103 - return self.apply(
1104 - prompt_text=processor_inputs.prompt_text,
1105 - mm_data=processor_inputs.mm_data,
1106 - hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1107 - )
1108 -
1109 - def get_dummy_data(self, seq_len: int) -> DummyData:
1110 - # Avoid circular import
1111 - from vllm.sequence import SequenceData
1112 -
1113 - profiling = self.profiling_info
1114 - mm_counts = profiling.get_mm_limits()
1115 - mm_max_tokens_per_item = profiling.get_mm_max_tokens_per_item(seq_len)
1116 - if mm_counts.keys() != mm_max_tokens_per_item.keys():
1117 - raise AssertionError(
1118 - "The keys returned by `get_supported_mm_limits`"
1119 - f"({set(mm_counts.keys())}) should be the same as those "
1120 - "returned by `get_mm_max_tokens_per_item` "
1121 - f"({set(mm_max_tokens_per_item.keys())})")
1122 -
1123 - mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
1124 - prompt_token_ids = mm_inputs["prompt_token_ids"]
1125 - placeholders_by_modality = mm_inputs["mm_placeholders"]
1126 -
1127 - total_placeholders_by_modality = {
1128 - modality: sum(item["length"] for item in placeholders)
1129 - for modality, placeholders in placeholders_by_modality.items()
1130 - }
1131 - expected_placeholders_by_modality = {
1132 - modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
1133 - for modality in placeholders_by_modality
1134 - }
1135 - if total_placeholders_by_modality != expected_placeholders_by_modality:
1136 - raise AssertionError(
1137 - f"The processed dummy data has a total of "
1138 - f"{total_placeholders_by_modality} placeholder tokens, which "
1139 - f"is not the expected {expected_placeholders_by_modality} "
1140 - "tokens.")
1141 -
1142 - total_len = len(prompt_token_ids)
1143 -
1144 - # V0 does not support chunked prefill.
1145 - if total_len > seq_len and not envs.VLLM_USE_V1:
1146 - logger.warning(
1147 - "The context length (%d) of the model is too short "
1148 - "to hold the multi-modal embeddings in the worst case "
1149 - "(%d tokens in total, out of which %s are reserved for "
1150 - "multi-modal embeddings). This may cause certain multi-modal "
1151 - "inputs to fail during inference, even when the input text is "
1152 - "short. To avoid this, you should increase `max_model_len`, "
1153 - "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
1154 - total_len, total_placeholders_by_modality)
1155 -
1156 - return DummyData(
1157 - seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
1158 - multi_modal_data=None,
1159 - multi_modal_placeholders=None,
1160 - )
1161 -
1162 - prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
1163 -
1164 - return DummyData(
1165 - seq_data=SequenceData.from_seqs(prompt_token_ids),
1166 - multi_modal_data=mm_inputs["mm_kwargs"],
1167 - multi_modal_placeholders=placeholders_by_modality,
1168 - )
vllm/multimodal/profiling.py
+117 -35
@@ -1,16 +1,18 @@
1 from abc import ABC, abstractmethod
2 from collections.abc import Mapping
3 from dataclasses import dataclass, field
4 -from typing import Optional
4 +from typing import Generic, TypeVar
5
6 import numpy as np
7 import numpy.typing as npt
8 from PIL import Image
9
10 -from vllm.inputs import InputProcessingContext
10 +import vllm.envs as envs
11 +from vllm.inputs import DummyData
12 from vllm.logger import init_logger
13
13 -from .inputs import MultiModalDataDict
14 +from .inputs import MultiModalDataDict, MultiModalInputsV2
15 +from .processing import BaseMultiModalProcessor, BaseProcessingInfo
16
17 logger = init_logger(__name__)
18
@@ -23,39 +25,19 @@ class ProcessorInputs:
25 hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
26
27
26 -class BaseProfilingInfo(ABC):
28 +_I = TypeVar("_I", bound=BaseProcessingInfo)
29 +
30 +
31 +class BaseDummyInputsBuilder(ABC, Generic[_I]):
32 """
28 - Abstract base class that provides the information necessary to profile
33 + Abstract base class that constructs the dummy data to profile
34 multi-modal models.
35 """
36
32 - def __init__(self, ctx: InputProcessingContext) -> None:
37 + def __init__(self, info: _I) -> None:
38 super().__init__()
39
35 - self.ctx = ctx
36 -
37 - @abstractmethod
38 - def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
39 - """
40 - Return the maximum supported number of items for each modality.
41 -
42 - A value of `None` means unlimited number of items.
43 -
44 - Omitting a modality from the returned dictionary means that
45 - it is not supported at all.
46 - """
47 - raise NotImplementedError
48 -
49 - @abstractmethod
50 - def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
51 - """
52 - Get the maximum possible number of tokens per data item
53 - for each modality.
54 -
55 - The dictionary returned by this method should have the same
56 - keys as that returned by :meth:`get_supported_mm_limits`.
57 - """
58 - raise NotImplementedError
40 + self.info = info
41
42 @abstractmethod
43 def get_dummy_processor_inputs(
@@ -64,8 +46,8 @@ class BaseProfilingInfo(ABC):
46 mm_counts: Mapping[str, int],
47 ) -> ProcessorInputs:
48 """
67 - Build the multi-modal portion of the input which, after processing,
68 - results in `mm_max_tokens` in :meth:`get_mm_max_tokens_per_item`.
49 + Build the input which, after processing, results in
50 + `self.info.get_mm_max_tokens_per_item()` placeholder tokens.
51 """
52 raise NotImplementedError
53
@@ -99,11 +81,33 @@ class BaseProfilingInfo(ABC):
81 video = np.zeros((num_frames, width, height, 3))
82 return [video] * num_videos
83
102 - def get_mm_limits(self) -> Mapping[str, int]:
103 - mm_config = self.ctx.get_mm_config()
84 +
85 +class MultiModalProfiler(Generic[_I]):
86 + """
87 + Contains code for running memory profiling for multi-modal models.
88 + """
89 +
90 + def __init__(
91 + self,
92 + processor: BaseMultiModalProcessor[_I],
93 + ) -> None:
94 + super().__init__()
95 +
96 + self.processor = processor
97 +
98 + @property
99 + def processing_info(self) -> BaseProcessingInfo:
100 + return self.processor.info
101 +
102 + @property
103 + def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
104 + return self.processor.dummy_inputs
105 +
106 + def _get_mm_limits(self) -> Mapping[str, int]:
107 + mm_config = self.processing_info.ctx.get_mm_config()
108 mm_limit_per_prompt = mm_config.limit_per_prompt
109
106 - supported_mm_limits = self.get_supported_mm_limits()
110 + supported_mm_limits = self.processing_info.get_supported_mm_limits()
111
112 mm_limits = {
113 modality: mm_limit_per_prompt.get(modality, 1)
@@ -119,3 +123,81 @@ class BaseProfilingInfo(ABC):
123 f"at most {supported_limit} {modality} items.")
124
125 return mm_limits
126 +
127 + def _get_dummy_mm_inputs(
128 + self,
129 + seq_len: int,
130 + mm_counts: Mapping[str, int],
131 + ) -> MultiModalInputsV2:
132 + factory = self.dummy_inputs
133 + processor_inputs = factory.get_dummy_processor_inputs(
134 + seq_len, mm_counts)
135 +
136 + return self.processor.apply(
137 + prompt_text=processor_inputs.prompt_text,
138 + mm_data=processor_inputs.mm_data,
139 + hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
140 + )
141 +
142 + def get_dummy_data(self, seq_len: int) -> DummyData:
143 + # Avoid circular import
144 + from vllm.sequence import SequenceData
145 +
146 + mm_counts = self._get_mm_limits()
147 +
148 + info = self.processing_info
149 + mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
150 +
151 + if mm_counts.keys() != mm_max_tokens_per_item.keys():
152 + raise AssertionError(
153 + "The keys returned by `get_supported_mm_limits`"
154 + f"({set(mm_counts.keys())}) should be the same as those "
155 + "returned by `get_mm_max_tokens_per_item` "
156 + f"({set(mm_max_tokens_per_item.keys())})")
157 +
158 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
159 + prompt_token_ids = mm_inputs["prompt_token_ids"]
160 + placeholders_by_modality = mm_inputs["mm_placeholders"]
161 +
162 + total_placeholders_by_modality = {
163 + modality: sum(item["length"] for item in placeholders)
164 + for modality, placeholders in placeholders_by_modality.items()
165 + }
166 + expected_placeholders_by_modality = {
167 + modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
168 + for modality in placeholders_by_modality
169 + }
170 + if total_placeholders_by_modality != expected_placeholders_by_modality:
171 + raise AssertionError(
172 + f"The processed dummy data has a total of "
173 + f"{total_placeholders_by_modality} placeholder tokens, which "
174 + f"is not the expected {expected_placeholders_by_modality} "
175 + "tokens.")
176 +
177 + total_len = len(prompt_token_ids)
178 +
179 + # V0 does not support chunked prefill.
180 + if total_len > seq_len and not envs.VLLM_USE_V1:
181 + logger.warning(
182 + "The context length (%d) of the model is too short "
183 + "to hold the multi-modal embeddings in the worst case "
184 + "(%d tokens in total, out of which %s are reserved for "
185 + "multi-modal embeddings). This may cause certain multi-modal "
186 + "inputs to fail during inference, even when the input text is "
187 + "short. To avoid this, you should increase `max_model_len`, "
188 + "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
189 + total_len, total_placeholders_by_modality)
190 +
191 + return DummyData(
192 + seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
193 + multi_modal_data=None,
194 + multi_modal_placeholders=None,
195 + )
196 +
197 + prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
198 +
199 + return DummyData(
200 + seq_data=SequenceData.from_seqs(prompt_token_ids),
201 + multi_modal_data=mm_inputs["mm_kwargs"],
202 + multi_modal_placeholders=placeholders_by_modality,
203 + )
vllm/multimodal/registry.py
+61 -12
@@ -1,7 +1,8 @@
1 import functools
2 from collections import UserDict
3 -from typing import (TYPE_CHECKING, Any, Dict, Mapping, Optional, Protocol,
4 - Sequence, Type, TypeVar)
3 +from dataclasses import dataclass
4 +from typing import (TYPE_CHECKING, Any, Dict, Generic, Mapping, Optional,
5 + Protocol, Sequence, Type, TypeVar)
6
7 import torch.nn as nn
8
@@ -14,7 +15,9 @@ from .audio import AudioPlugin
15 from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
16 from .image import ImagePlugin
17 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
17 -from .processing import BaseMultiModalProcessor, ProcessingCache
18 +from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
19 + ProcessingCache)
20 +from .profiling import BaseDummyInputsBuilder
21 from .utils import cached_get_tokenizer
22 from .video import VideoPlugin
23
@@ -27,20 +30,59 @@ logger = init_logger(__name__)
30 MM_CACHE_SIZE = 256
31
32 N = TypeVar("N", bound=Type[nn.Module])
33 +_I = TypeVar("_I", bound=BaseProcessingInfo)
34 +_I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
35
36
32 -class MultiModalProcessorFactory(Protocol):
37 +class ProcessingInfoFactory(Protocol[_I_co]):
38 """Constructs a :class:`MultiModalProcessor` instance from the context."""
39
40 def __call__(
41 self,
42 ctx: InputProcessingContext,
43 + ) -> _I_co:
44 + ...
45 +
46 +
47 +class DummyInputsBuilderFactory(Protocol[_I]):
48 + """
49 + Constructs a :class:`BaseDummyInputsBuilder` instance from the context.
50 + """
51 +
52 + def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]:
53 + ...
54 +
55 +
56 +class MultiModalProcessorFactory(Protocol[_I]):
57 + """Constructs a :class:`MultiModalProcessor` instance from the context."""
58 +
59 + def __call__(
60 + self,
61 + info: _I,
62 + dummy_inputs: BaseDummyInputsBuilder[_I],
63 *,
64 cache: Optional[ProcessingCache] = None,
40 - ) -> BaseMultiModalProcessor:
65 + ) -> BaseMultiModalProcessor[_I]:
66 ...
67
68
69 +@dataclass(frozen=True)
70 +class _ProcessorFactories(Generic[_I]):
71 + info: ProcessingInfoFactory[_I]
72 + processor: MultiModalProcessorFactory[_I]
73 + dummy_inputs: DummyInputsBuilderFactory[_I]
74 +
75 + def build_processor(
76 + self,
77 + ctx: InputProcessingContext,
78 + *,
79 + cache: Optional[ProcessingCache] = None,
80 + ):
81 + info = self.info(ctx)
82 + dummy_inputs_builder = self.dummy_inputs(info)
83 + return self.processor(info, dummy_inputs_builder, cache=cache)
84 +
85 +
86 class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
87 """
88 Wraps `_limits_by_model` for a more informative error message
@@ -71,7 +113,7 @@ class MultiModalRegistry:
113 self._plugins = {p.get_data_key(): p for p in plugins}
114
115 self._processor_factories = ClassRegistry[nn.Module,
74 - MultiModalProcessorFactory]()
116 + _ProcessorFactories]()
117
118 # This is used for non-multimodal models
119 self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
@@ -224,7 +266,7 @@ class MultiModalRegistry:
266 tokenizer = cached_get_tokenizer(model_config.tokenizer)
267 processor = self.create_processor(model_config, tokenizer)
268 seq_len = model_config.max_model_len
227 - return processor.profiling_info.get_mm_max_tokens_per_item(seq_len)
269 + return processor.info.get_mm_max_tokens_per_item(seq_len)
270
271 return {
272 key: plugin.get_max_multimodal_tokens(model_config)
@@ -315,7 +357,10 @@ class MultiModalRegistry:
357
358 def register_processor(
359 self,
318 - factory: MultiModalProcessorFactory,
360 + processor: MultiModalProcessorFactory[_I],
361 + *,
362 + info: ProcessingInfoFactory[_I],
363 + dummy_inputs: DummyInputsBuilderFactory[_I],
364 ):
365 """
366 Register a multi-modal processor to a model class. The processor
@@ -336,7 +381,11 @@ class MultiModalRegistry:
381 "registered to %s. It is overwritten by the new one.",
382 model_cls, self)
383
339 - self._processor_factories[model_cls] = factory
384 + self._processor_factories[model_cls] = _ProcessorFactories(
385 + info=info,
386 + dummy_inputs=dummy_inputs,
387 + processor=processor,
388 + )
389
390 return model_cls
391
@@ -359,15 +408,15 @@ class MultiModalRegistry:
408 self,
409 model_config: "ModelConfig",
410 tokenizer: AnyTokenizer,
362 - ) -> BaseMultiModalProcessor:
411 + ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
412 """
413 Create a multi-modal processor for a specific model and tokenizer.
414 """
415 model_cls = self._get_model_cls(model_config)
367 - processor_factory = self._processor_factories[model_cls]
416 + factories = self._processor_factories[model_cls]
417
418 ctx = InputProcessingContext(model_config, tokenizer)
419 cache = (None if model_config.disable_mm_preprocessor_cache else
420 self._processing_cache)
421
373 - return processor_factory(ctx, cache=cache)
422 + return factories.build_processor(ctx, cache=cache)