A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Reorganize profiling/processing-related code (#11812)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 8, 2025 at 18:59 UTC
2a0596bc480bb835dc05a30f5e708ecbfffbcd69
23 files changed
+833
-760
tests/models/decoder_only/vision_language/processing/test_llava_next.py
+14
-27
@@ -4,24 +4,17 @@ from functools import partial
4
import pytest
5
from PIL import Image
6
from pqdm.threads import pqdm
7
-from transformers import AutoTokenizer
7
9
-from vllm.inputs import InputProcessingContext
8
+from vllm.multimodal import MULTIMODAL_REGISTRY
9
from vllm.multimodal.parse import ImageSize
10
+from vllm.multimodal.processing import BaseMultiModalProcessor
11
+from vllm.multimodal.utils import cached_get_tokenizer
12
13
from ....utils import build_model_context
14
15
15
-# Fixtures lazy import to avoid initializing CUDA during test collection
16
-@pytest.fixture()
17
-def processor_for_llava_next():
18
- from vllm.model_executor.models.llava_next import (
19
- LlavaNextMultiModalProcessor)
20
- return LlavaNextMultiModalProcessor
21
-
22
-
16
def _validate_image_prompt_replacements_one(
24
- processor,
17
+ processor: BaseMultiModalProcessor,
18
num_imgs: int,
19
failed_size_excs: list[tuple[ImageSize, Exception]],
20
image_size: ImageSize,
@@ -78,20 +71,17 @@ def _test_image_prompt_replacements(
71
72
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
73
@pytest.mark.parametrize("num_imgs", [1, 2])
81
-def test_processor_prompt_replacements_regression(
82
- processor_for_llava_next,
83
- model_id: str,
84
- num_imgs: int,
85
-):
74
+def test_processor_prompt_replacements_regression(model_id, num_imgs):
75
ctx = build_model_context(
76
model_name=model_id,
77
tokenizer_name=model_id,
78
mm_processor_kwargs=None,
79
limit_mm_per_prompt={"image": num_imgs},
80
)
92
- tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
93
- ctx = InputProcessingContext(ctx.model_config, tokenizer)
94
- processor = processor_for_llava_next(ctx)
81
+ processor = MULTIMODAL_REGISTRY.create_processor(
82
+ ctx.model_config,
83
+ tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
84
+ )
85
86
image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
87
(488, 183), (2560, 1669)]
@@ -111,20 +101,17 @@ def test_processor_prompt_replacements_regression(
101
"Comment this out to run it manually.")
102
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
103
@pytest.mark.parametrize("num_imgs", [1])
114
-def test_processor_prompt_replacements_all(
115
- processor_for_llava_next,
116
- model_id: str,
117
- num_imgs: int,
118
-):
104
+def test_processor_prompt_replacements_all(model_id, num_imgs):
105
ctx = build_model_context(
106
model_name=model_id,
107
tokenizer_name=model_id,
108
mm_processor_kwargs=None,
109
limit_mm_per_prompt={"image": num_imgs},
110
)
125
- tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126
- ctx = InputProcessingContext(ctx.model_config, tokenizer)
127
- processor = processor_for_llava_next(ctx)
111
+ processor = MULTIMODAL_REGISTRY.create_processor(
112
+ ctx.model_config,
113
+ tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
114
+ )
115
116
seen_aspect_ratios = set[float]()
117
image_sizes = list[ImageSize]()
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py
+14
-27
@@ -4,24 +4,17 @@ from functools import partial
4
import pytest
5
from PIL import Image
6
from pqdm.threads import pqdm
7
-from transformers import AutoTokenizer
7
9
-from vllm.inputs import InputProcessingContext
8
+from vllm.multimodal import MULTIMODAL_REGISTRY
9
from vllm.multimodal.parse import ImageSize
10
+from vllm.multimodal.processing import BaseMultiModalProcessor
11
+from vllm.multimodal.utils import cached_get_tokenizer
12
13
from ....utils import build_model_context
14
15
15
-# Fixtures lazy import to avoid initializing CUDA during test collection
16
-@pytest.fixture()
17
-def processor_for_llava_onevision():
18
- from vllm.model_executor.models.llava_onevision import (
19
- LlavaOnevisionMultiModalProcessor)
20
- return LlavaOnevisionMultiModalProcessor
21
-
22
-
16
def _validate_image_prompt_replacements_one(
24
- processor,
17
+ processor: BaseMultiModalProcessor,
18
num_imgs: int,
19
failed_size_excs: list[tuple[ImageSize, Exception]],
20
image_size: ImageSize,
@@ -77,20 +70,17 @@ def _test_image_prompt_replacements(
70
@pytest.mark.parametrize("model_id",
71
["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
72
@pytest.mark.parametrize("num_imgs", [1, 2])
80
-def test_processor_prompt_replacements_regression(
81
- processor_for_llava_onevision,
82
- model_id: str,
83
- num_imgs: int,
84
-):
73
+def test_processor_prompt_replacements_regression(model_id, num_imgs):
74
ctx = build_model_context(
75
model_name=model_id,
76
tokenizer_name=model_id,
77
mm_processor_kwargs=None,
78
limit_mm_per_prompt={"image": num_imgs},
79
)
91
- tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
92
- ctx = InputProcessingContext(ctx.model_config, tokenizer)
93
- processor = processor_for_llava_onevision(ctx)
80
+ processor = MULTIMODAL_REGISTRY.create_processor(
81
+ ctx.model_config,
82
+ tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
83
+ )
84
85
image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
86
(488, 183), (2560, 1669)]
@@ -111,20 +101,17 @@ def test_processor_prompt_replacements_regression(
101
@pytest.mark.parametrize("model_id",
102
["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
103
@pytest.mark.parametrize("num_imgs", [1])
114
-def test_processor_prompt_replacements_all(
115
- processor_for_llava_onevision,
116
- model_id: str,
117
- num_imgs: int,
118
-):
104
+def test_processor_prompt_replacements_all(model_id, num_imgs):
105
ctx = build_model_context(
106
model_name=model_id,
107
tokenizer_name=model_id,
108
mm_processor_kwargs=None,
109
limit_mm_per_prompt={"image": num_imgs},
110
)
125
- tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
126
- ctx = InputProcessingContext(ctx.model_config, tokenizer)
127
- processor = processor_for_llava_onevision(ctx)
111
+ processor = MULTIMODAL_REGISTRY.create_processor(
112
+ ctx.model_config,
113
+ tokenizer=cached_get_tokenizer(ctx.model_config.tokenizer),
114
+ )
115
116
seen_aspect_ratios = set[float]()
117
image_sizes = list[ImageSize]()
tests/models/decoder_only/vision_language/processing/test_phi3v.py
+10
-14
@@ -1,21 +1,13 @@
1
"""Tests for phi3v's multimodal preprocessing kwargs."""
2
import pytest
3
-from transformers import AutoTokenizer
3
5
-from vllm.inputs import InputProcessingContext
6
-from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
4
+from vllm.multimodal import MULTIMODAL_REGISTRY
5
+from vllm.multimodal.utils import cached_get_tokenizer
6
7
from .....conftest import _ImageAssets
8
from ....utils import build_model_context
9
10
12
-# Wrap lazy imports to avoid initializing CUDA during test collection
13
-@pytest.fixture()
14
-def processor_for_phi3v():
15
- from vllm.model_executor.models.phi3v import Phi3VMultiModalProcessor
16
- return Phi3VMultiModalProcessor
17
-
18
-
11
@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
12
# yapf: disable
13
@pytest.mark.parametrize(
@@ -29,7 +21,6 @@ def processor_for_phi3v():
21
# yapf: enable
22
@pytest.mark.parametrize("num_imgs", [1, 2])
23
def test_processor_override(
32
- processor_for_phi3v,
24
image_assets: _ImageAssets,
25
model_id: str,
26
mm_processor_kwargs: dict[str, int],
@@ -37,21 +28,26 @@ def test_processor_override(
28
num_imgs: int,
29
):
30
"""Ensure input_processor_for_phi3v handles num_crops properly."""
31
+ # Avoid initializing CUDA early
32
+ from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
33
+
34
ctx = build_model_context(
35
model_name=model_id,
36
tokenizer_name=model_id,
37
trust_remote_code=True,
38
limit_mm_per_prompt={"image": num_imgs},
39
)
46
- tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
47
- ctx = InputProcessingContext(ctx.model_config, tokenizer)
40
+ tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
41
+ processor = MULTIMODAL_REGISTRY.create_processor(
42
+ ctx.model_config,
43
+ tokenizer=tokenizer,
44
+ )
45
46
# Build the image str / prompt based on the number of images we pass
47
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
48
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
49
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
50
54
- processor = processor_for_phi3v(ctx)
51
processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
52
53
# Ensure we have the right number of placeholders per num_crops size
tests/models/decoder_only/vision_language/processing/test_qwen2_vl.py
+8
-14
@@ -1,19 +1,12 @@
1
import pytest
2
-from transformers import AutoTokenizer
2
4
-from vllm.inputs import InputProcessingContext
3
+from vllm.multimodal import MULTIMODAL_REGISTRY
4
+from vllm.multimodal.utils import cached_get_tokenizer
5
6
from .....conftest import _ImageAssets
7
from ....utils import build_model_context
8
9
10
-# Fixtures lazy import to avoid initializing CUDA during test collection
11
-@pytest.fixture()
12
-def processor_for_qwen2_vl():
13
- from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalProcessor
14
- return Qwen2VLMultiModalProcessor
15
-
16
-
10
@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
11
# yapf: disable
12
@pytest.mark.parametrize(
@@ -24,7 +17,6 @@ def processor_for_qwen2_vl():
17
# yapf: enable
18
@pytest.mark.parametrize("num_imgs", [1, 2])
19
def test_processor_override(
27
- processor_for_qwen2_vl,
20
image_assets: _ImageAssets,
21
model_id: str,
22
mm_processor_kwargs: dict[str, object],
@@ -39,18 +31,20 @@ def test_processor_override(
31
mm_processor_kwargs=None,
32
limit_mm_per_prompt={"image": num_imgs},
33
)
42
- tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
43
- ctx = InputProcessingContext(ctx.model_config, tokenizer)
34
+ tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
35
+ processor = MULTIMODAL_REGISTRY.create_processor(
36
+ ctx.model_config,
37
+ tokenizer=tokenizer,
38
+ )
39
40
# Build the image str / prompt based on the number of images we pass
41
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
42
mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
43
49
- processor = processor_for_qwen2_vl(ctx)
44
processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
45
46
# Ensure we have the right number of placeholders per num_crops size
53
- hf_processor = processor._get_hf_processor(**mm_processor_kwargs)
47
+ hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
48
image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
49
img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
50
pixel_shape = processed_inputs["mm_kwargs"]["pixel_values"].shape
tests/multimodal/test_processing.py
+25
-27
@@ -10,12 +10,17 @@ from PIL import Image
10
from vllm.config import ModelConfig
11
from vllm.inputs import InputProcessingContext
12
from vllm.multimodal import MULTIMODAL_REGISTRY
13
-from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
14
- _PlaceholderInfo, find_mm_placeholders,
13
+# yapf conflicts with isort for this block
14
+# yapf: disable
15
+from vllm.multimodal.processing import (PlaceholderInfo, ProcessingCache,
16
+ PromptReplacement,
17
+ find_mm_placeholders,
18
find_text_matches, find_token_matches,
19
iter_token_matches,
20
replace_text_matches,
21
replace_token_matches)
22
+# yapf: enable
23
+from vllm.multimodal.profiling import MultiModalProfiler
24
from vllm.multimodal.utils import cached_get_tokenizer
25
from vllm.transformers_utils.tokenizer import AnyTokenizer
26
from vllm.utils import full_groupby
@@ -431,7 +436,7 @@ def test_find_replace_tokens(
436
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
437
{
438
"pattern_1": [
434
- _PlaceholderInfo(
439
+ PlaceholderInfo(
440
modality="pattern_1",
441
item_idx=0,
442
start_idx=6,
@@ -445,13 +450,13 @@ def test_find_replace_tokens(
450
[1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
451
{
452
"pattern_1": [
448
- _PlaceholderInfo(
453
+ PlaceholderInfo(
454
modality="pattern_1",
455
item_idx=0,
456
start_idx=1,
457
replacement=[32000, 32000],
458
),
454
- _PlaceholderInfo(
459
+ PlaceholderInfo(
460
modality="pattern_1",
461
item_idx=1,
462
start_idx=5,
@@ -459,7 +464,7 @@ def test_find_replace_tokens(
464
),
465
],
466
"pattern_3": [
462
- _PlaceholderInfo(
467
+ PlaceholderInfo(
468
modality="pattern_3",
469
item_idx=0,
470
start_idx=7,
@@ -472,13 +477,13 @@ def test_find_replace_tokens(
477
[1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
478
{
479
"pattern_1": [
475
- _PlaceholderInfo(
480
+ PlaceholderInfo(
481
modality="pattern_1",
482
item_idx=0,
483
start_idx=1,
484
replacement=[32000, 32000],
485
),
481
- _PlaceholderInfo(
486
+ PlaceholderInfo(
487
modality="pattern_1",
488
item_idx=1,
489
start_idx=3,
@@ -486,7 +491,7 @@ def test_find_replace_tokens(
491
),
492
],
493
"pattern_3": [
489
- _PlaceholderInfo(
494
+ PlaceholderInfo(
495
modality="pattern_3",
496
item_idx=0,
497
start_idx=6,
@@ -577,19 +582,15 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
582
revision=None,
583
limit_mm_per_prompt=limit_mm_per_prompt,
584
)
580
- model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
585
582
- processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
583
- ctx = InputProcessingContext(
586
+ processor = MULTIMODAL_REGISTRY.create_processor(
587
model_config,
588
tokenizer=cached_get_tokenizer(model_config.tokenizer),
589
)
587
-
588
- processor = processor_factory(ctx, cache=None)
589
- profiler = processor.profiling_info
590
+ profiler = MultiModalProfiler(processor)
591
592
mock_supported_mm_limits = MagicMock(return_value={"image": num_supported})
592
- profiler.get_supported_mm_limits = mock_supported_mm_limits
593
+ processor.info.get_supported_mm_limits = mock_supported_mm_limits
594
595
if is_valid:
596
exc_ctx = nullcontext()
@@ -597,7 +598,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
598
exc_ctx = pytest.raises(ValueError, match="this model only supports")
599
600
with exc_ctx:
600
- profiler.get_mm_limits()
601
+ profiler.get_dummy_data(model_config.max_model_len)
602
603
604
@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
@@ -620,16 +621,12 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
621
revision=None,
622
limit_mm_per_prompt=limit_mm_per_prompt,
623
)
623
- model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
624
625
- processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
626
- ctx = InputProcessingContext(
625
+ processor = MULTIMODAL_REGISTRY.create_processor(
626
model_config,
627
tokenizer=cached_get_tokenizer(model_config.tokenizer),
628
)
629
631
- processor = processor_factory(ctx, cache=None)
632
-
630
rng = np.random.RandomState(0)
631
image = _rand_img(rng, min_wh=128, max_wh=256)
632
if num_images == 0:
@@ -681,9 +678,9 @@ def _test_processing_cache_correctness(
678
hf_overrides=hf_overrides,
679
limit_mm_per_prompt=limit_mm_per_prompt,
680
)
684
- model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
681
686
- processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
682
+ model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
683
+ factories = MULTIMODAL_REGISTRY._processor_factories[model_cls]
684
ctx = InputProcessingContext(
685
model_config,
686
tokenizer=cached_get_tokenizer(model_config.tokenizer),
@@ -691,8 +688,9 @@ def _test_processing_cache_correctness(
688
# Ensure that it can fit all of the data
689
cache = ProcessingCache(capacity=1 << 30)
690
694
- baseline_processor = processor_factory(ctx, cache=None)
695
- cached_processor = processor_factory(ctx, cache=cache)
691
+ baseline_processor = factories.build_processor(ctx, cache=None)
692
+ cached_processor = factories.build_processor(ctx, cache=cache)
693
+ dummy_inputs = baseline_processor.dummy_inputs
694
695
rng = np.random.RandomState(0)
696
@@ -724,7 +722,7 @@ def _test_processing_cache_correctness(
722
}
723
724
mm_counts = {k: len(vs) for k, vs in mm_data.items()}
727
- prompt = baseline_processor.profiling_info.get_dummy_processor_inputs(
725
+ prompt = dummy_inputs.get_dummy_processor_inputs(
726
model_config.max_model_len,
727
mm_counts,
728
).prompt_text
tests/plugins/vllm_add_dummy_model/vllm_add_dummy_model/my_llava.py
+7
-3
@@ -2,13 +2,17 @@ from typing import Optional
2
3
import torch
4
5
-from vllm.model_executor.models.llava import (LlavaForConditionalGeneration,
6
- LlavaMultiModalProcessor)
5
+from vllm.model_executor.models.llava import (LlavaDummyInputsBuilder,
6
+ LlavaForConditionalGeneration,
7
+ LlavaMultiModalProcessor,
8
+ LlavaProcessingInfo)
9
from vllm.model_executor.sampling_metadata import SamplingMetadata
10
from vllm.multimodal import MULTIMODAL_REGISTRY
11
12
11
-@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor)
13
+@MULTIMODAL_REGISTRY.register_processor(LlavaMultiModalProcessor,
14
+ info=LlavaProcessingInfo,
15
+ dummy_inputs=LlavaDummyInputsBuilder)
16
class MyLlava(LlavaForConditionalGeneration):
17
18
def compute_logits(
vllm/inputs/preprocess.py
+1
-1
@@ -7,7 +7,7 @@ from vllm.config import ModelConfig
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
9
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
10
-from vllm.multimodal.processing import MultiModalDataDict, MultiModalInputsV2
10
+from vllm.multimodal.inputs import MultiModalDataDict, MultiModalInputsV2
11
from vllm.prompt_adapter.request import PromptAdapterRequest
12
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
13
from vllm.utils import print_info_once, print_warning_once
vllm/inputs/registry.py
+3
-1
@@ -323,6 +323,7 @@ class InputRegistry:
323
# Avoid circular import
324
from vllm.model_executor.model_loader import get_model_architecture
325
from vllm.multimodal import MultiModalKwargs
326
+ from vllm.multimodal.profiling import MultiModalProfiler
327
from vllm.multimodal.utils import cached_get_tokenizer
328
329
if mm_registry.has_processor(model_config):
@@ -331,7 +332,8 @@ class InputRegistry:
332
trust_remote_code=model_config.trust_remote_code,
333
)
334
processor = mm_registry.create_processor(model_config, tokenizer)
334
- dummy_data = processor.get_dummy_data(seq_len)
335
+ profiler = MultiModalProfiler(processor)
336
+ dummy_data = profiler.get_dummy_data(seq_len)
337
else:
338
model_cls, _ = get_model_architecture(model_config)
339
if is_encoder_data:
vllm/model_executor/models/aria.py
+23
-24
@@ -23,10 +23,10 @@ from vllm.model_executor.model_loader.weight_utils import (
23
from vllm.multimodal import MULTIMODAL_REGISTRY
24
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
25
NestedTensors)
26
+from vllm.multimodal.parse import MultiModalDataItems
27
from vllm.multimodal.processing import (BaseMultiModalProcessor,
27
- MultiModalDataItems, ProcessingMixin,
28
- PromptReplacement)
29
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
28
+ BaseProcessingInfo, PromptReplacement)
29
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30
from vllm.sequence import IntermediateTensors
31
from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
32
AriaVisionConfig)
@@ -445,33 +445,33 @@ def build_mm_projector(config: PretrainedConfig):
445
)
446
447
448
-class AriaProcessingMixin(ProcessingMixin):
448
+class AriaProcessingInfo(BaseProcessingInfo):
449
450
- def _get_hf_config(self):
450
+ def get_hf_config(self):
451
return self.ctx.get_hf_config()
452
453
- def _get_vision_config(self) -> AriaVisionConfig:
454
- return self._get_hf_config().vision_config
455
-
456
- def _get_num_image_tokens(self) -> int:
457
- hf_config = self._get_hf_config()
458
- return max(hf_config.projector_patch_to_query_dict.values())
459
-
460
-
461
-class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
453
+ def get_vision_config(self) -> AriaVisionConfig:
454
+ return self.get_hf_config().vision_config
455
456
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
457
return {"image": None}
458
459
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
467
- return {"image": self._get_num_image_tokens()}
460
+ return {"image": self.get_num_image_tokens()}
461
+
462
+ def get_num_image_tokens(self) -> int:
463
+ hf_config = self.get_hf_config()
464
+ return max(hf_config.projector_patch_to_query_dict.values())
465
+
466
+
467
+class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
468
469
def get_dummy_processor_inputs(
470
self,
471
seq_len: int,
472
mm_counts: Mapping[str, int],
473
) -> ProcessorInputs:
474
- vision_config = self._get_vision_config()
474
+ vision_config = self.info.get_vision_config()
475
476
max_image_size = vision_config.image_size
477
num_images = mm_counts.get("image", 0)
@@ -483,7 +483,7 @@ class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
483
num_images=num_images)
484
}
485
486
- hf_processor = self._get_hf_processor()
486
+ hf_processor = self.info.get_hf_processor()
487
image_token: str = hf_processor.image_token # type: ignore
488
489
return ProcessorInputs(
@@ -492,10 +492,7 @@ class AriaProfilingInfo(AriaProcessingMixin, BaseProfilingInfo):
492
)
493
494
495
-class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
496
-
497
- def _get_profiling_info(self) -> BaseProfilingInfo:
498
- return AriaProfilingInfo(self.ctx)
495
+class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
496
497
def _get_mm_fields_config(
498
self,
@@ -513,10 +510,10 @@ class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
510
hf_processor_mm_kwargs: Mapping[str, object],
511
out_mm_kwargs: MultiModalKwargs,
512
) -> list[PromptReplacement]:
516
- hf_config = self._get_hf_config()
513
+ hf_config = self.info.get_hf_config()
514
image_token_id = hf_config.image_token_index
515
519
- num_image_tokens = self._get_num_image_tokens()
516
+ num_image_tokens = self.info.get_num_image_tokens()
517
518
return [
519
PromptReplacement(
@@ -527,7 +524,9 @@ class AriaMultiModalProcessor(AriaProcessingMixin, BaseMultiModalProcessor):
524
]
525
526
530
-@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
527
+@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor,
528
+ info=AriaProcessingInfo,
529
+ dummy_inputs=AriaDummyInputsBuilder)
530
class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
531
"""
532
Aria model for conditional generation tasks.
vllm/model_executor/models/blip2.py
+19
-20
@@ -17,10 +17,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
17
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
18
MultiModalInputsV2, MultiModalKwargs,
19
NestedTensors, PlaceholderRange)
20
+from vllm.multimodal.parse import MultiModalDataItems
21
from vllm.multimodal.processing import (BaseMultiModalProcessor,
21
- MultiModalDataItems, ProcessingMixin,
22
- PromptReplacement)
23
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
22
+ BaseProcessingInfo, PromptReplacement)
23
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
24
from vllm.sequence import IntermediateTensors
25
26
from .blip import BlipVisionModel
@@ -397,30 +397,30 @@ class Blip2QFormerModel(nn.Module):
397
return sequence_output
398
399
400
-class Blip2ProcessingMixin(ProcessingMixin):
400
+class Blip2ProcessingInfo(BaseProcessingInfo):
401
402
- def _get_hf_config(self):
402
+ def get_hf_config(self):
403
return self.ctx.get_hf_config(Blip2Config)
404
405
- def _get_num_image_tokens(self) -> int:
406
- hf_config = self._get_hf_config()
407
- return hf_config.num_query_tokens
408
-
409
-
410
-class Blip2ProfilingInfo(Blip2ProcessingMixin, BaseProfilingInfo):
411
-
405
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
406
return {"image": 1}
407
408
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
416
- return {"image": self._get_num_image_tokens()}
409
+ return {"image": self.get_num_image_tokens()}
410
+
411
+ def get_num_image_tokens(self) -> int:
412
+ hf_config = self.get_hf_config()
413
+ return hf_config.num_query_tokens
414
+
415
+
416
+class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
417
418
def get_dummy_processor_inputs(
419
self,
420
seq_len: int,
421
mm_counts: Mapping[str, int],
422
) -> ProcessorInputs:
423
- hf_config = self._get_hf_config()
423
+ hf_config = self.info.get_hf_config()
424
vision_config = hf_config.vision_config
425
426
max_image_size = vision_config.image_size
@@ -439,10 +439,7 @@ class Blip2ProfilingInfo(Blip2ProcessingMixin, BaseProfilingInfo):
439
)
440
441
442
-class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
443
-
444
- def _get_profiling_info(self) -> BaseProfilingInfo:
445
- return Blip2ProfilingInfo(self.ctx)
442
+class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
443
444
def _get_mm_fields_config(
445
self,
@@ -460,7 +457,7 @@ class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
457
hf_processor_mm_kwargs: Mapping[str, object],
458
out_mm_kwargs: MultiModalKwargs,
459
) -> list[PromptReplacement]:
463
- num_image_tokens = self._get_num_image_tokens()
460
+ num_image_tokens = self.info.get_num_image_tokens()
461
462
return [
463
PromptReplacement(
@@ -491,7 +488,9 @@ class Blip2MultiModalProcessor(Blip2ProcessingMixin, BaseMultiModalProcessor):
488
return result
489
490
494
-@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
491
+@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor,
492
+ info=Blip2ProcessingInfo,
493
+ dummy_inputs=Blip2DummyInputsBuilder)
494
class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
495
496
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/chameleon.py
+24
-23
@@ -30,10 +30,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
30
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
31
MultiModalInputsV2, MultiModalKwargs,
32
NestedTensors, PlaceholderRange)
33
+from vllm.multimodal.parse import MultiModalDataItems
34
from vllm.multimodal.processing import (BaseMultiModalProcessor,
34
- MultiModalDataItems, ProcessingMixin,
35
- PromptReplacement)
36
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
35
+ BaseProcessingInfo, PromptReplacement)
36
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37
from vllm.sequence import IntermediateTensors
38
from vllm.utils import print_warning_once
39
@@ -49,33 +49,34 @@ class ChameleonImagePixelInputs(TypedDict):
49
"""Shape: `(batch_size * num_images, num_channels, height, width)`"""
50
51
52
-class ChameleonProcessingMixin(ProcessingMixin):
52
+class ChameleonProcessingInfo(BaseProcessingInfo):
53
54
- def _get_hf_config(self):
54
+ def get_hf_config(self):
55
return self.ctx.get_hf_config(ChameleonConfig)
56
57
- def _get_hf_processor(self):
57
+ def get_hf_processor(self):
58
return self.ctx.get_hf_processor(ChameleonProcessor)
59
60
- def _get_num_image_tokens(self) -> int:
61
- processor = self._get_hf_processor()
62
- return processor.image_seq_length
63
-
64
-
65
-class ChameleonProfilingInfo(ChameleonProcessingMixin, BaseProfilingInfo):
66
-
60
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
61
return {"image": 1}
62
63
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
71
- return {"image": self._get_num_image_tokens()}
64
+ return {"image": self.get_num_image_tokens()}
65
+
66
+ def get_num_image_tokens(self) -> int:
67
+ processor = self.get_hf_processor()
68
+ return processor.image_seq_length
69
+
70
+
71
+class ChameleonDummyInputsBuilder(
72
+ BaseDummyInputsBuilder[ChameleonProcessingInfo]):
73
74
def get_dummy_processor_inputs(
75
self,
76
seq_len: int,
77
mm_counts: Mapping[str, int],
78
) -> ProcessorInputs:
78
- config = self._get_hf_config()
79
+ config = self.info.get_hf_config()
80
81
width = height = config.vq_config.resolution
82
num_images = mm_counts.get("image", 0)
@@ -93,11 +94,8 @@ class ChameleonProfilingInfo(ChameleonProcessingMixin, BaseProfilingInfo):
94
)
95
96
96
-class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
97
- BaseMultiModalProcessor):
98
-
99
- def _get_profiling_info(self) -> BaseProfilingInfo:
100
- return ChameleonProfilingInfo(self.ctx)
97
+class ChameleonMultiModalProcessor(
98
+ BaseMultiModalProcessor[ChameleonProcessingInfo]):
99
100
def _get_mm_fields_config(
101
self,
@@ -112,7 +110,7 @@ class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
110
hf_processor_mm_kwargs: Mapping[str, object],
111
out_mm_kwargs: MultiModalKwargs,
112
) -> list[PromptReplacement]:
115
- processor = self._get_hf_processor(**hf_processor_mm_kwargs)
113
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
114
115
return [
116
PromptReplacement(
@@ -120,7 +118,7 @@ class ChameleonMultiModalProcessor(ChameleonProcessingMixin,
118
target="<image>",
119
replacement="".join([
120
processor.image_start_token,
123
- processor.image_token * self._get_num_image_tokens(),
121
+ processor.image_token * self.info.get_num_image_tokens(),
122
processor.image_end_token,
123
]),
124
)
@@ -916,7 +914,10 @@ class ChameleonModel(nn.Module):
914
return hidden_states
915
916
919
-@MULTIMODAL_REGISTRY.register_processor(ChameleonMultiModalProcessor)
917
+@MULTIMODAL_REGISTRY.register_processor(
918
+ ChameleonMultiModalProcessor,
919
+ info=ChameleonProcessingInfo,
920
+ dummy_inputs=ChameleonDummyInputsBuilder)
921
class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
922
SupportsPP):
923
vllm/model_executor/models/fuyu.py
+40
-40
@@ -33,11 +33,11 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
33
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
MultiModalInputsV2, MultiModalKwargs,
35
NestedTensors, PlaceholderRange)
36
-from vllm.multimodal.parse import ImageProcessorItems, ImageSize
36
+from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37
+ MultiModalDataItems)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
- MultiModalDataItems, ProcessingMixin,
39
- PromptReplacement)
40
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
39
+ BaseProcessingInfo, PromptReplacement)
40
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
41
from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsMultiModal, SupportsPP
@@ -64,24 +64,38 @@ class FuyuImagePatchInputs(TypedDict):
64
"""
65
66
67
-class FuyuProcessingMixin(ProcessingMixin):
67
+class FuyuProcessingInfo(BaseProcessingInfo):
68
69
- def _get_hf_config(self):
69
+ def get_hf_config(self):
70
return self.ctx.get_hf_config(FuyuConfig)
71
72
- def _get_hf_processor(self):
72
+ def get_hf_processor(self):
73
return self.ctx.get_hf_processor(FuyuProcessor)
74
75
- def _get_image_processor(self) -> FuyuImageProcessor:
76
- return self._get_hf_processor().image_processor
75
+ def get_image_processor(self) -> FuyuImageProcessor:
76
+ return self.get_hf_processor().image_processor
77
78
- def _get_image_feature_grid_size(
78
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
79
+ return {"image": 1}
80
+
81
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
82
+ target_width, target_height = self.get_image_size_with_most_features()
83
+
84
+ max_ncols, max_nrows = self.get_image_feature_grid_size(
85
+ image_width=target_width,
86
+ image_height=target_height,
87
+ )
88
+ max_image_tokens = (max_ncols + 1) * max_nrows
89
+
90
+ return {"image": max_image_tokens}
91
+
92
+ def get_image_feature_grid_size(
93
self,
94
*,
95
image_width: int,
96
image_height: int,
97
) -> tuple[int, int]:
84
- image_processor = self._get_image_processor()
98
+ image_processor = self.get_image_processor()
99
target_width = image_processor.size["width"]
100
target_height = image_processor.size["height"]
101
@@ -97,34 +111,21 @@ class FuyuProcessingMixin(ProcessingMixin):
111
nrows = math.ceil(image_height / 30)
112
return ncols, nrows
113
100
-
101
-class FuyuProfilingInfo(FuyuProcessingMixin, BaseProfilingInfo):
102
-
103
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104
- return {"image": 1}
105
-
106
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
107
- target_width, target_height = self._get_image_size_with_most_features()
108
-
109
- max_ncols, max_nrows = self._get_image_feature_grid_size(
110
- image_width=target_width,
111
- image_height=target_height,
112
- )
113
- max_image_tokens = (max_ncols + 1) * max_nrows
114
-
115
- return {"image": max_image_tokens}
116
-
117
- def _get_image_size_with_most_features(self) -> ImageSize:
118
- image_processor = self._get_image_processor()
114
+ def get_image_size_with_most_features(self) -> ImageSize:
115
+ image_processor = self.get_image_processor()
116
return ImageSize(width=image_processor.size["width"],
117
height=image_processor.size["height"])
118
119
+
120
+class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
121
+
122
def get_dummy_processor_inputs(
123
self,
124
seq_len: int,
125
mm_counts: Mapping[str, int],
126
) -> ProcessorInputs:
127
- target_width, target_height = self._get_image_size_with_most_features()
127
+ target_width, target_height = \
128
+ self.info.get_image_size_with_most_features()
129
num_images = mm_counts.get("image", 0)
130
131
mm_data = {
@@ -140,10 +141,7 @@ class FuyuProfilingInfo(FuyuProcessingMixin, BaseProfilingInfo):
141
)
142
143
143
-class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
144
-
145
- def _get_profiling_info(self) -> BaseProfilingInfo:
146
- return FuyuProfilingInfo(self.ctx)
144
+class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
145
146
def _call_hf_processor(
147
self,
@@ -156,7 +154,7 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
154
# Avoid warning from HF logger for text-only input
155
# Input_ids format: bos_token_id + prompt_token_ids + boa_token_id
156
# Tokenizer won't add boa_token_id by default, we add it manually.
159
- tokenizer = self._get_tokenizer()
157
+ tokenizer = self.info.get_tokenizer()
158
boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
159
prompt_ids = tokenizer.encode(prompt) + [boa_token_id]
160
return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
@@ -196,10 +194,10 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
194
hf_processor_mm_kwargs: Mapping[str, object],
195
out_mm_kwargs: MultiModalKwargs,
196
) -> list[PromptReplacement]:
199
- hf_config = self._get_hf_config()
197
+ hf_config = self.info.get_hf_config()
198
bos_token_id = hf_config.bos_token_id
199
202
- tokenizer = self._get_tokenizer()
200
+ tokenizer = self.info.get_tokenizer()
201
eot_token_id = tokenizer.bos_token_id
202
assert isinstance(eot_token_id, int)
203
@@ -207,7 +205,7 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
205
images = mm_items.get_items("image", ImageProcessorItems)
206
image_size = images.get_image_size(item_idx)
207
210
- ncols, nrows = self._get_image_feature_grid_size(
208
+ ncols, nrows = self.info.get_image_feature_grid_size(
209
image_width=image_size.width,
210
image_height=image_size.height,
211
)
@@ -244,7 +242,9 @@ class FuyuMultiModalProcessor(FuyuProcessingMixin, BaseMultiModalProcessor):
242
return result
243
244
247
-@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
245
+@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor,
246
+ info=FuyuProcessingInfo,
247
+ dummy_inputs=FuyuDummyInputsBuilder)
248
class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
249
250
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/llava.py
+89
-86
@@ -1,7 +1,7 @@
1
-from abc import ABC, abstractmethod
1
+from abc import abstractmethod
2
from functools import cached_property
3
from typing import (Final, Iterable, List, Literal, Mapping, Optional,
4
- Protocol, Set, Tuple, TypedDict, Union)
4
+ Protocol, Set, Tuple, TypedDict, TypeVar, Union)
5
6
import torch
7
import torch.nn as nn
@@ -25,11 +25,11 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25
MultiModalInputsV2, MultiModalKwargs,
26
NestedTensors)
27
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28
- ImageSize)
28
+ ImageSize, MultiModalDataItems)
29
from vllm.multimodal.processing import (BaseMultiModalProcessor,
30
- MultiModalDataItems, ProcessingCache,
31
- ProcessingMixin, PromptReplacement)
32
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
30
+ BaseProcessingInfo, ProcessingCache,
31
+ PromptReplacement)
32
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
33
from vllm.sequence import IntermediateTensors
34
35
from .clip import CLIPVisionModel
@@ -105,34 +105,23 @@ class LlavaLikeProcessor(Protocol):
105
image_token: Final[str]
106
107
108
-class BaseLlavaProcessingMixin(ProcessingMixin, ABC):
108
+class BaseLlavaProcessingInfo(BaseProcessingInfo):
109
110
- def _get_hf_config(self) -> LlavaLikeConfig:
110
+ def get_hf_config(self) -> LlavaLikeConfig:
111
return self.ctx.get_hf_config(LlavaConfig)
112
113
- def _get_vision_encoder_info(self):
114
- return get_vision_encoder_info(self._get_hf_config())
113
+ def get_vision_encoder_info(self):
114
+ return get_vision_encoder_info(self.get_hf_config())
115
116
@abstractmethod
117
- def _get_hf_processor(self) -> LlavaLikeProcessor:
117
+ def get_hf_processor(self) -> LlavaLikeProcessor:
118
raise NotImplementedError
119
120
- def _get_num_image_tokens(
121
- self,
122
- *,
123
- image_width: int,
124
- image_height: int,
125
- ) -> int:
126
- hf_config = self._get_hf_config()
127
- vision_encoder_info = self._get_vision_encoder_info()
120
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
121
+ return {"image": None}
122
129
- return self._apply_feature_select_strategy(
130
- hf_config.vision_feature_select_strategy,
131
- vision_encoder_info.get_num_image_tokens(
132
- image_width=image_width,
133
- image_height=image_height,
134
- ),
135
- )
123
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
124
+ return {"image": self.get_max_image_tokens()}
125
126
def _apply_feature_select_strategy(
127
self,
@@ -147,28 +136,42 @@ class BaseLlavaProcessingMixin(ProcessingMixin, ABC):
136
msg = f"Unexpected feature select strategy: {strategy!r}"
137
raise NotImplementedError(msg)
138
139
+ def get_num_image_tokens(
140
+ self,
141
+ *,
142
+ image_width: int,
143
+ image_height: int,
144
+ ) -> int:
145
+ hf_config = self.get_hf_config()
146
+ vision_encoder_info = self.get_vision_encoder_info()
147
151
-class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
152
-
153
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
154
- return {"image": None}
155
-
156
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
157
- return {"image": self._get_max_image_tokens()}
148
+ return self._apply_feature_select_strategy(
149
+ hf_config.vision_feature_select_strategy,
150
+ vision_encoder_info.get_num_image_tokens(
151
+ image_width=image_width,
152
+ image_height=image_height,
153
+ ),
154
+ )
155
159
- def _get_image_size_with_most_features(self) -> ImageSize:
160
- vision_encoder_info = self._get_vision_encoder_info()
156
+ def get_image_size_with_most_features(self) -> ImageSize:
157
+ vision_encoder_info = self.get_vision_encoder_info()
158
width = height = vision_encoder_info.get_image_size()
159
return ImageSize(width=width, height=height)
160
164
- def _get_max_image_tokens(self) -> int:
165
- target_width, target_height = self._get_image_size_with_most_features()
161
+ def get_max_image_tokens(self) -> int:
162
+ target_width, target_height = self.get_image_size_with_most_features()
163
167
- return self._get_num_image_tokens(
164
+ return self.get_num_image_tokens(
165
image_width=target_width,
166
image_height=target_height,
167
)
168
169
+
170
+_I = TypeVar("_I", bound=BaseLlavaProcessingInfo)
171
+
172
+
173
+class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
174
+
175
def get_dummy_processor_inputs(
176
self,
177
seq_len: int,
@@ -176,9 +179,10 @@ class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
179
) -> ProcessorInputs:
180
num_images = mm_counts.get("image", 0)
181
179
- processor = self._get_hf_processor()
182
+ processor = self.info.get_hf_processor()
183
image_token = processor.image_token
181
- target_width, target_height = self._get_image_size_with_most_features()
184
+ target_width, target_height = \
185
+ self.info.get_image_size_with_most_features()
186
187
mm_data = {
188
"image":
@@ -193,23 +197,13 @@ class BaseLlavaProfilingInfo(BaseLlavaProcessingMixin, BaseProfilingInfo):
197
)
198
199
196
-class LlavaProcessingMixin(BaseLlavaProcessingMixin):
200
+class LlavaProcessingInfo(BaseLlavaProcessingInfo):
201
198
- def _get_hf_processor(self):
202
+ def get_hf_processor(self):
203
return self.ctx.get_hf_processor(LlavaProcessor)
204
205
202
-class LlavaProfilingInfo(LlavaProcessingMixin, BaseLlavaProfilingInfo):
203
- pass
204
-
205
-
206
-class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
207
- BaseMultiModalProcessor):
208
-
209
- # Copied from BaseMultiModalProcessor
210
- @abstractmethod
211
- def _get_profiling_info(self) -> BaseProfilingInfo:
212
- raise NotImplementedError
206
+class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
207
208
# Copied from BaseMultiModalProcessor
209
@abstractmethod
@@ -226,7 +220,7 @@ class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
220
hf_processor_mm_kwargs: Mapping[str, object],
221
out_mm_kwargs: MultiModalKwargs,
222
) -> list[PromptReplacement]:
229
- hf_config = self._get_hf_config()
223
+ hf_config = self.info.get_hf_config()
224
image_token_id = hf_config.image_token_index
225
226
def get_replacement(item_idx: int):
@@ -237,7 +231,7 @@ class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
231
num_image_tokens = images.get_feature_size(item_idx)
232
else:
233
image_size = images.get_image_size(item_idx)
240
- num_image_tokens = self._get_num_image_tokens(
234
+ num_image_tokens = self.info.get_num_image_tokens(
235
image_width=image_size.width,
236
image_height=image_size.height,
237
)
@@ -253,10 +247,8 @@ class BaseLlavaMultiModalProcessor(LlavaProcessingMixin,
247
]
248
249
256
-class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
257
-
258
- def _get_profiling_info(self) -> BaseProfilingInfo:
259
- return LlavaProfilingInfo(self.ctx)
250
+class LlavaMultiModalProcessor(
251
+ BaseLlavaMultiModalProcessor[LlavaProcessingInfo]):
252
253
def _get_mm_fields_config(
254
self,
@@ -269,21 +261,14 @@ class LlavaMultiModalProcessor(BaseLlavaMultiModalProcessor):
261
)
262
263
272
-class PixtralHFProcessingMixin(BaseLlavaProcessingMixin):
264
+class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
265
274
- def _get_hf_processor(self):
266
+ def get_hf_processor(self):
267
return self.ctx.get_hf_processor(PixtralProcessor)
268
269
278
-class PixtralHFProfilingInfo(PixtralHFProcessingMixin, BaseLlavaProfilingInfo):
279
- pass
280
-
281
-
282
-class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
283
- BaseMultiModalProcessor):
284
-
285
- def _get_profiling_info(self) -> BaseProfilingInfo:
286
- return PixtralHFProfilingInfo(self.ctx)
270
+class PixtralHFMultiModalProcessor(
271
+ BaseMultiModalProcessor[PixtralHFProcessingInfo]):
272
273
def _call_hf_processor(
274
self,
@@ -328,10 +313,10 @@ class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
313
hf_processor_mm_kwargs: Mapping[str, object],
314
out_mm_kwargs: MultiModalKwargs,
315
) -> list[PromptReplacement]:
331
- hf_config = self._get_hf_config()
316
+ hf_config = self.info.get_hf_config()
317
image_token_id = hf_config.image_token_index
318
334
- processor = self._get_hf_processor()
319
+ processor = self.info.get_hf_processor()
320
image_token = processor.image_token
321
image_break_token = processor.image_break_token
322
image_end_token = processor.image_end_token
@@ -363,26 +348,40 @@ class PixtralHFMultiModalProcessor(PixtralHFProcessingMixin,
348
]
349
350
351
+def _build_llava_or_pixtral_hf_info(
352
+ ctx: InputProcessingContext, ) -> BaseLlavaProcessingInfo:
353
+ hf_config = ctx.get_hf_config(LlavaConfig)
354
+
355
+ if isinstance(hf_config.vision_config, PixtralVisionConfig):
356
+ return PixtralHFProcessingInfo(ctx)
357
+
358
+ return LlavaProcessingInfo(ctx)
359
+
360
+
361
def _build_llava_or_pixtral_hf_processor(
367
- ctx: InputProcessingContext,
362
+ info: _I,
363
+ dummy_inputs: BaseDummyInputsBuilder[_I],
364
*,
365
cache: Optional[ProcessingCache] = None,
366
enable_sanity_checks: bool = True,
367
) -> BaseMultiModalProcessor:
372
- hf_config = ctx.get_hf_config(LlavaConfig)
373
-
374
- if isinstance(hf_config.vision_config, PixtralVisionConfig):
368
+ if isinstance(info, PixtralHFProcessingInfo):
369
return PixtralHFMultiModalProcessor(
376
- ctx,
370
+ info,
371
+ dummy_inputs, # type: ignore
372
+ cache=cache,
373
+ enable_sanity_checks=enable_sanity_checks,
374
+ )
375
+
376
+ if isinstance(info, LlavaProcessingInfo):
377
+ return LlavaMultiModalProcessor(
378
+ info,
379
+ dummy_inputs, # type: ignore
380
cache=cache,
381
enable_sanity_checks=enable_sanity_checks,
382
)
383
381
- return LlavaMultiModalProcessor(
382
- ctx,
383
- cache=cache,
384
- enable_sanity_checks=enable_sanity_checks,
385
- )
384
+ raise NotImplementedError(type(info))
385
386
387
def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
@@ -460,7 +459,9 @@ def init_vision_tower_for_llava(
459
raise NotImplementedError(msg)
460
461
463
-@MULTIMODAL_REGISTRY.register_processor(_build_llava_or_pixtral_hf_processor)
462
+@MULTIMODAL_REGISTRY.register_processor(_build_llava_or_pixtral_hf_processor,
463
+ info=_build_llava_or_pixtral_hf_info,
464
+ dummy_inputs=LlavaDummyInputsBuilder)
465
class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
466
# BitandBytes specific attributes
467
bitsandbytes_stacked_params_mapping = {
@@ -727,11 +728,11 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
728
mm_data: MultiModalDataDict,
729
hf_processor_mm_kwargs: Mapping[str, object],
730
) -> MultiModalInputsV2:
730
- hf_config = self._get_hf_config()
731
+ hf_config = self.info.get_hf_config()
732
image_token_id = hf_config.image_token_index
733
734
# Assume that it doesn't depend on the image size
734
- num_image_tokens = self._get_num_image_tokens(
735
+ num_image_tokens = self.info.get_num_image_tokens(
736
image_width=-1,
737
image_height=-1,
738
)
@@ -796,6 +797,8 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
797
798
# To use this model, please use
799
# `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'`
799
-@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor)
800
+@MULTIMODAL_REGISTRY.register_processor(MantisMultiModalProcessor,
801
+ info=LlavaProcessingInfo,
802
+ dummy_inputs=LlavaDummyInputsBuilder)
803
class MantisForConditionalGeneration(LlavaForConditionalGeneration):
804
pass
vllm/model_executor/models/llava_next.py
+33
-22
@@ -1,6 +1,7 @@
1
+from abc import abstractmethod
2
from functools import cached_property
3
from typing import (Final, Iterable, List, Literal, Mapping, Optional,
3
- Protocol, Set, Tuple, TypedDict, Union)
4
+ Protocol, Set, Tuple, TypedDict, TypeVar, Union)
5
6
import torch
7
import torch.nn as nn
@@ -16,13 +17,12 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors
19
from vllm.multimodal.parse import ImageSize
19
-from vllm.multimodal.profiling import BaseProfilingInfo
20
from vllm.sequence import IntermediateTensors
21
22
from .clip import CLIPVisionModel
23
from .interfaces import SupportsMultiModal, SupportsPP
24
-from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingMixin,
25
- BaseLlavaProfilingInfo, LlavaLikeConfig,
24
+from .llava import (BaseLlavaMultiModalProcessor, BaseLlavaProcessingInfo,
25
+ LlavaDummyInputsBuilder, LlavaLikeConfig,
26
LlavaMultiModalProjector, init_vision_tower_for_llava)
27
from .siglip import SiglipVisionModel
28
from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,
@@ -65,23 +65,23 @@ class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
65
image_grid_pinpoints: Final[list[list[int]]]
66
67
68
-class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
68
+class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
69
70
- def _get_hf_config(self) -> LlavaNextLikeConfig:
70
+ def get_hf_config(self) -> LlavaNextLikeConfig:
71
return self.ctx.get_hf_config(LlavaNextConfig)
72
73
- def _get_hf_processor(self):
73
+ def get_hf_processor(self):
74
return self.ctx.get_hf_processor(LlavaNextProcessor)
75
76
# Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L113
77
- def _get_num_image_tokens(
77
+ def get_num_image_tokens(
78
self,
79
*,
80
image_width: int,
81
image_height: int,
82
) -> int:
83
- hf_config = self._get_hf_config()
84
- vision_encoder_info = self._get_vision_encoder_info()
83
+ hf_config = self.get_hf_config()
84
+ vision_encoder_info = self.get_vision_encoder_info()
85
86
base_feature_size = self._apply_feature_select_strategy(
87
hf_config.vision_feature_select_strategy,
@@ -140,16 +140,13 @@ class LlavaNextProcessingMixin(BaseLlavaProcessingMixin):
140
141
return (unpadded_features, newline_features)
142
143
-
144
-class LlavaNextProfilingInfo(LlavaNextProcessingMixin, BaseLlavaProfilingInfo):
145
-
146
- def _get_image_size_with_most_features(self) -> ImageSize:
147
- hf_config = self._get_hf_config()
143
+ def get_image_size_with_most_features(self) -> ImageSize:
144
+ hf_config = self.get_hf_config()
145
146
largest_feature_size, largest_feature_pinpoint = 0, None
147
for (height, width) in hf_config.image_grid_pinpoints:
151
- feat_size = self._get_num_image_tokens(image_width=width,
152
- image_height=height)
148
+ feat_size = self.get_num_image_tokens(image_width=width,
149
+ image_height=height)
150
if feat_size > largest_feature_size:
151
largest_feature_size = feat_size
152
largest_feature_pinpoint = ImageSize(width=width,
@@ -161,11 +158,23 @@ class LlavaNextProfilingInfo(LlavaNextProcessingMixin, BaseLlavaProfilingInfo):
158
return largest_feature_pinpoint
159
160
164
-class LlavaNextMultiModalProcessor(LlavaNextProcessingMixin,
165
- BaseLlavaMultiModalProcessor):
161
+_I = TypeVar("_I", bound=LlavaNextProcessingInfo)
162
+
163
+
164
+class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]):
165
+
166
+ # Copied from BaseMultiModalProcessor
167
+ @abstractmethod
168
+ def _get_mm_fields_config(
169
+ self,
170
+ hf_inputs: BatchFeature,
171
+ hf_processor_mm_kwargs: Mapping[str, object],
172
+ ) -> Mapping[str, MultiModalFieldConfig]:
173
+ raise NotImplementedError
174
+
175
167
- def _get_profiling_info(self) -> BaseProfilingInfo:
168
- return LlavaNextProfilingInfo(self.ctx)
176
+class LlavaNextMultiModalProcessor(
177
+ BaseLlavaNextMultiModalProcessor[LlavaNextProcessingInfo]):
178
179
def _get_mm_fields_config(
180
self,
@@ -179,7 +188,9 @@ class LlavaNextMultiModalProcessor(LlavaNextProcessingMixin,
188
)
189
190
182
-@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor)
191
+@MULTIMODAL_REGISTRY.register_processor(LlavaNextMultiModalProcessor,
192
+ info=LlavaNextProcessingInfo,
193
+ dummy_inputs=LlavaDummyInputsBuilder)
194
class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
195
SupportsPP):
196
vllm/model_executor/models/llava_next_video.py
+54
-50
@@ -17,12 +17,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
19
NestedTensors)
20
-from vllm.multimodal.parse import (ImageSize, VideoEmbeddingItems,
21
- VideoProcessorItems)
20
+from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
21
+ VideoEmbeddingItems, VideoProcessorItems)
22
from vllm.multimodal.processing import (BaseMultiModalProcessor,
23
- MultiModalDataItems, ProcessingMixin,
24
- PromptReplacement)
25
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
23
+ BaseProcessingInfo, PromptReplacement)
24
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25
from vllm.sequence import IntermediateTensors
26
from vllm.utils import is_list_of
27
@@ -47,33 +46,52 @@ class LlavaNextVideoPixelInputs(TypedDict):
46
"""
47
48
50
-class LlavaNextVideoProcessingMixin(ProcessingMixin):
49
+class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
50
52
- def _get_hf_config(self):
51
+ def get_hf_config(self):
52
return self.ctx.get_hf_config(LlavaNextVideoConfig)
53
55
- def _get_vision_encoder_info(self):
56
- return get_vision_encoder_info(self._get_hf_config())
54
+ def get_vision_encoder_info(self):
55
+ return get_vision_encoder_info(self.get_hf_config())
56
58
- def _get_hf_processor(self):
57
+ def get_hf_processor(self):
58
return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
59
60
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
61
+ return {"video": 1}
62
+
63
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
64
+ target_width, target_height = self.get_image_size_with_most_features()
65
+
66
+ max_video_tokens = self.get_num_video_tokens(
67
+ image_width=target_width,
68
+ image_height=target_height,
69
+ num_frames=self.get_num_frames_with_most_features(seq_len),
70
+ )
71
+
72
+ return {"video": max_video_tokens}
73
+
74
+ def get_image_size_with_most_features(self) -> ImageSize:
75
+ vision_encoder_info = self.get_vision_encoder_info()
76
+ width = height = vision_encoder_info.get_image_size()
77
+ return ImageSize(width=width, height=height)
78
+
79
def _get_num_frame_tokens(
80
self,
81
*,
82
image_width: int,
83
image_height: int,
84
) -> int:
67
- hf_config = self._get_hf_config()
85
+ hf_config = self.get_hf_config()
86
spatial_pool_stride = hf_config.spatial_pool_stride
87
70
- vision_encoder_info = self._get_vision_encoder_info()
88
+ vision_encoder_info = self.get_vision_encoder_info()
89
patch_grid_length = vision_encoder_info.get_patch_grid_length()
90
pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
91
92
return pooled_grid_length * pooled_grid_length
93
76
- def _get_num_video_tokens(
94
+ def get_num_video_tokens(
95
self,
96
*,
97
image_width: int,
@@ -87,37 +105,14 @@ class LlavaNextVideoProcessingMixin(ProcessingMixin):
105
106
return num_frame_tokens * num_frames
107
90
-
91
-class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
92
- BaseProfilingInfo):
93
-
94
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
95
- return {"video": 1}
96
-
97
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
98
- target_width, target_height = self._get_image_size_with_most_features()
99
-
100
- max_video_tokens = self._get_num_video_tokens(
101
- image_width=target_width,
102
- image_height=target_height,
103
- num_frames=self._get_dummy_num_frames(seq_len),
104
- )
105
-
106
- return {"video": max_video_tokens}
107
-
108
- def _get_image_size_with_most_features(self) -> ImageSize:
109
- vision_encoder_info = self._get_vision_encoder_info()
110
- width = height = vision_encoder_info.get_image_size()
111
- return ImageSize(width=width, height=height)
112
-
108
def _get_max_video_frames(self, max_tokens: int) -> int:
114
- target_width, target_height = self._get_image_size_with_most_features()
109
+ target_width, target_height = self.get_image_size_with_most_features()
110
111
num_frames = 0
112
113
while True:
114
next_num_frames = num_frames + 1
120
- next_max_tokens = self._get_num_video_tokens(
115
+ next_max_tokens = self.get_num_video_tokens(
116
image_width=target_width,
117
image_height=target_height,
118
num_frames=next_num_frames,
@@ -130,7 +125,7 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
125
126
return num_frames
127
133
- def _get_dummy_num_frames(self, seq_len: int) -> int:
128
+ def get_num_frames_with_most_features(self, seq_len: int) -> int:
129
mm_config = self.ctx.get_mm_config()
130
max_videos = mm_config.limit_per_prompt.get("video", 1)
131
@@ -138,6 +133,10 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
133
134
return max(max_total_frames // max(max_videos, 1), 1)
135
136
+
137
+class LlavaNextVideoDummyInputsBuilder(
138
+ BaseDummyInputsBuilder[LlavaNextVideoProcessingInfo]):
139
+
140
def get_dummy_processor_inputs(
141
self,
142
seq_len: int,
@@ -145,16 +144,20 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
144
) -> ProcessorInputs:
145
num_videos = mm_counts.get("video", 0)
146
148
- processor = self._get_hf_processor()
147
+ processor = self.info.get_hf_processor()
148
video_token = processor.video_token
150
- target_width, target_height = self._get_image_size_with_most_features()
149
+
150
+ target_width, target_height = \
151
+ self.info.get_image_size_with_most_features()
152
+ target_num_frames = \
153
+ self.info.get_num_frames_with_most_features(seq_len)
154
155
mm_data = {
156
"video":
157
self._get_dummy_videos(
158
width=target_width,
159
height=target_height,
157
- num_frames=self._get_dummy_num_frames(seq_len),
160
+ num_frames=target_num_frames,
161
num_videos=num_videos,
162
)
163
}
@@ -165,11 +168,8 @@ class LlavaNextVideoProfilingInfo(LlavaNextVideoProcessingMixin,
168
)
169
170
168
-class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
169
- BaseMultiModalProcessor):
170
-
171
- def _get_profiling_info(self) -> BaseProfilingInfo:
172
- return LlavaNextVideoProfilingInfo(self.ctx)
171
+class LlavaNextVideoMultiModalProcessor(
172
+ BaseMultiModalProcessor[LlavaNextVideoProcessingInfo]):
173
174
def _get_mm_fields_config(
175
self,
@@ -184,7 +184,7 @@ class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
184
hf_processor_mm_kwargs: Mapping[str, object],
185
out_mm_kwargs: MultiModalKwargs,
186
) -> list[PromptReplacement]:
187
- hf_config = self._get_hf_config()
187
+ hf_config = self.info.get_hf_config()
188
video_token_id = hf_config.video_token_index
189
190
def get_replacement(item_idx: int):
@@ -195,7 +195,7 @@ class LlavaNextVideoMultiModalProcessor(LlavaNextVideoProcessingMixin,
195
num_video_tokens = videos.get_feature_size(item_idx)
196
else:
197
image_size = videos.get_frame_size(item_idx)
198
- num_video_tokens = self._get_num_video_tokens(
198
+ num_video_tokens = self.info.get_num_video_tokens(
199
image_width=image_size.width,
200
image_height=image_size.height,
201
num_frames=videos.get_num_frames(item_idx),
@@ -269,7 +269,11 @@ class LlavaNextMultiModalProjector(nn.Module):
269
return hidden_states
270
271
272
-@MULTIMODAL_REGISTRY.register_processor(LlavaNextVideoMultiModalProcessor)
272
+@MULTIMODAL_REGISTRY.register_processor(
273
+ LlavaNextVideoMultiModalProcessor,
274
+ info=LlavaNextVideoProcessingInfo,
275
+ dummy_inputs=LlavaNextVideoDummyInputsBuilder,
276
+)
277
class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
278
SupportsPP):
279
vllm/model_executor/models/llava_onevision.py
+52
-63
@@ -17,19 +17,20 @@ from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
20
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
21
-from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
22
- VideoEmbeddingItems, VideoProcessorItems)
23
-from vllm.multimodal.processing import MultiModalFieldConfig, PromptReplacement
24
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
20
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
21
+ NestedTensors)
22
+from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
23
+ VideoProcessorItems)
24
+from vllm.multimodal.processing import PromptReplacement
25
+from vllm.multimodal.profiling import ProcessorInputs
26
from vllm.sequence import IntermediateTensors
27
from vllm.utils import is_list_of
28
29
from .clip import CLIPVisionModel
30
from .interfaces import SupportsMultiModal, SupportsPP
30
-from .llava import BaseLlavaProfilingInfo, init_vision_tower_for_llava
31
-from .llava_next import (LlavaNextLikeConfig, LlavaNextMultiModalProcessor,
32
- LlavaNextProcessingMixin)
31
+from .llava import LlavaDummyInputsBuilder, init_vision_tower_for_llava
32
+from .llava_next import (BaseLlavaNextMultiModalProcessor, LlavaNextLikeConfig,
33
+ LlavaNextProcessingInfo)
34
from .siglip import SiglipVisionModel
35
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
36
maybe_prefix, merge_multimodal_embeddings)
@@ -89,14 +90,23 @@ class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
90
video_token_index: Final[int]
91
92
92
-class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
93
+class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
94
94
- def _get_hf_config(self) -> LlavaOnevisionLikeConfig:
95
+ def get_hf_config(self) -> LlavaOnevisionLikeConfig:
96
return self.ctx.get_hf_config(LlavaOnevisionConfig)
97
97
- def _get_hf_processor(self):
98
+ def get_hf_processor(self):
99
return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
100
101
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
102
+ return {"image": None, "video": None}
103
+
104
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
105
+ return {
106
+ "image": self.get_max_image_tokens(),
107
+ "video": self.get_max_video_tokens(seq_len),
108
+ }
109
+
110
# Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
111
# with additional logic afterwards taken from LlavaOnevisionProcessor
112
def _get_num_unpadded_features(
@@ -141,16 +151,16 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
151
image_width: int,
152
image_height: int,
153
) -> int:
144
- hf_config = self._get_hf_config()
154
+ hf_config = self.get_hf_config()
155
spatial_pool_stride = getattr(hf_config, "spatial_pool_stride", 2)
156
147
- vision_encoder_info = self._get_vision_encoder_info()
157
+ vision_encoder_info = self.get_vision_encoder_info()
158
patch_grid_length = vision_encoder_info.get_patch_grid_length()
159
pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
160
161
return pooled_grid_length * pooled_grid_length
162
153
- def _get_num_video_tokens(
163
+ def get_num_video_tokens(
164
self,
165
*,
166
image_width: int,
@@ -164,43 +174,14 @@ class LlavaOnevisionProcessingMixin(LlavaNextProcessingMixin):
174
175
return num_frame_tokens * num_frames + 1 # Newline token
176
167
-
168
-class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
169
- BaseLlavaProfilingInfo):
170
-
171
- def _get_image_size_with_most_features(self) -> ImageSize:
172
- hf_config = self._get_hf_config()
173
- largest_feature_size, largest_feature_pinpoint = 0, None
174
- for (height, width) in hf_config.image_grid_pinpoints:
175
- feat_size = self._get_num_image_tokens(image_width=width,
176
- image_height=height)
177
- if feat_size > largest_feature_size:
178
- largest_feature_size = feat_size
179
- largest_feature_pinpoint = ImageSize(width=width,
180
- height=height)
181
-
182
- if largest_feature_size == 0 or largest_feature_pinpoint is None:
183
- raise ValueError("Cannot have a largest feature size of 0!")
184
-
185
- return largest_feature_pinpoint
186
-
187
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
188
- return {"image": None, "video": None}
189
-
190
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
191
- return {
192
- "image": self._get_max_image_tokens(),
193
- "video": self._get_max_video_tokens(seq_len),
194
- }
195
-
177
def _get_max_video_frames(self, max_tokens: int) -> int:
197
- target_width, target_height = self._get_image_size_with_most_features()
178
+ target_width, target_height = self.get_image_size_with_most_features()
179
180
num_frames = 0
181
182
while True:
183
next_num_frames = num_frames + 1
203
- next_max_tokens = self._get_num_video_tokens(
184
+ next_max_tokens = self.get_num_video_tokens(
185
image_width=target_width,
186
image_height=target_height,
187
num_frames=next_num_frames,
@@ -213,12 +194,12 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
194
195
return num_frames
196
216
- def _get_dummy_num_frames(self, seq_len: int) -> int:
197
+ def get_num_frames_with_most_features(self, seq_len: int) -> int:
198
mm_config = self.ctx.get_mm_config()
199
max_images = mm_config.limit_per_prompt.get("image", 1)
200
max_videos = mm_config.limit_per_prompt.get("video", 1)
201
221
- max_image_tokens = self._get_max_image_tokens() * max_images
202
+ max_image_tokens = self.get_max_image_tokens() * max_images
203
max_total_frames = self._get_max_video_frames(seq_len -
204
max_image_tokens)
205
max_frames_per_video = min(max_total_frames // max(max_videos, 1),
@@ -226,15 +207,19 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
207
208
return max(max_frames_per_video, 1)
209
229
- def _get_max_video_tokens(self, seq_len: int) -> int:
230
- target_width, target_height = self._get_image_size_with_most_features()
210
+ def get_max_video_tokens(self, seq_len: int) -> int:
211
+ target_width, target_height = self.get_image_size_with_most_features()
212
232
- return self._get_num_video_tokens(
213
+ return self.get_num_video_tokens(
214
image_width=target_width,
215
image_height=target_height,
235
- num_frames=self._get_dummy_num_frames(seq_len),
216
+ num_frames=self.get_num_frames_with_most_features(seq_len),
217
)
218
219
+
220
+class LlavaOnevisionDummyInputsBuilder(
221
+ LlavaDummyInputsBuilder[LlavaOnevisionProcessingInfo]):
222
+
223
def get_dummy_processor_inputs(
224
self,
225
seq_len: int,
@@ -243,10 +228,14 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
228
num_images = mm_counts.get("image", 0)
229
num_videos = mm_counts.get("video", 0)
230
246
- processor = self._get_hf_processor()
231
+ processor = self.info.get_hf_processor()
232
image_token = processor.image_token
233
video_token = processor.video_token
249
- target_width, target_height = self._get_image_size_with_most_features()
234
+
235
+ target_width, target_height = \
236
+ self.info.get_image_size_with_most_features()
237
+ target_num_frames = \
238
+ self.info.get_num_frames_with_most_features(seq_len)
239
240
mm_data = {
241
"image":
@@ -257,7 +246,7 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
246
self._get_dummy_videos(
247
width=target_width,
248
height=target_height,
260
- num_frames=self._get_dummy_num_frames(seq_len),
249
+ num_frames=target_num_frames,
250
num_videos=num_videos,
251
)
252
}
@@ -268,11 +257,8 @@ class LlavaOnevisionProfilingInfo(LlavaOnevisionProcessingMixin,
257
)
258
259
271
-class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
272
- LlavaNextMultiModalProcessor):
273
-
274
- def _get_profiling_info(self) -> BaseProfilingInfo:
275
- return LlavaOnevisionProfilingInfo(self.ctx)
260
+class LlavaOnevisionMultiModalProcessor(
261
+ BaseLlavaNextMultiModalProcessor[LlavaOnevisionProcessingInfo]):
262
263
def _get_mm_fields_config(
264
self,
@@ -303,7 +289,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
289
mm_kwargs=mm_kwargs,
290
)
291
306
- processor = self._get_hf_processor()
292
+ processor = self.info.get_hf_processor()
293
video_token = processor.video_token
294
295
# LLaVA-OneVision processor doesn't support multiple videos
@@ -345,7 +331,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
331
out_mm_kwargs=out_mm_kwargs,
332
)
333
348
- hf_config = self._get_hf_config()
334
+ hf_config = self.info.get_hf_config()
335
video_token_id = hf_config.video_token_index
336
337
def get_video_replacement(item_idx: int):
@@ -356,7 +342,7 @@ class LlavaOnevisionMultiModalProcessor(LlavaOnevisionProcessingMixin,
342
num_video_tokens = videos.get_feature_size(item_idx)
343
else:
344
image_size = videos.get_frame_size(item_idx)
359
- num_video_tokens = self._get_num_video_tokens(
345
+ num_video_tokens = self.info.get_num_video_tokens(
346
image_width=image_size.width,
347
image_height=image_size.height,
348
num_frames=videos.get_num_frames(item_idx),
@@ -393,7 +379,10 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
379
return hidden_states
380
381
396
-@MULTIMODAL_REGISTRY.register_processor(LlavaOnevisionMultiModalProcessor)
382
+@MULTIMODAL_REGISTRY.register_processor(
383
+ LlavaOnevisionMultiModalProcessor,
384
+ info=LlavaOnevisionProcessingInfo,
385
+ dummy_inputs=LlavaOnevisionDummyInputsBuilder)
386
class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
387
SupportsPP):
388
vllm/model_executor/models/phi3v.py
+43
-40
@@ -34,13 +34,12 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
34
MultiModalInputsV2, MultiModalKwargs,
35
NestedTensors, PlaceholderRange)
36
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
37
- ImageSize)
37
+ ImageSize, MultiModalDataItems)
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
- MultiModalDataItems, ProcessingMixin,
40
- PromptReplacement,
41
- _BoundPromptReplacement,
42
- _PlaceholderInfo)
43
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
39
+ BaseProcessingInfo,
40
+ BoundPromptReplacement,
41
+ PlaceholderInfo, PromptReplacement)
42
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
43
from vllm.sequence import IntermediateTensors
44
from vllm.utils import is_list_of
45
@@ -302,9 +301,9 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
301
return image_features_hd_newline
302
303
305
-class Phi3VProcessingMixin(ProcessingMixin):
304
+class Phi3VProcessingInfo(BaseProcessingInfo):
305
307
- def _get_hf_processor(
306
+ def get_hf_processor(
307
self,
308
*,
309
num_crops: Optional[int] = None,
@@ -314,39 +313,42 @@ class Phi3VProcessingMixin(ProcessingMixin):
313
314
return self.ctx.get_hf_processor()
315
317
- def _get_num_image_tokens(
318
- self,
319
- *,
320
- image_width: int,
321
- image_height: int,
322
- ) -> int:
323
- processor = self._get_hf_processor()
324
-
325
- return processor.calc_num_image_tokens_from_image_size( # type: ignore
326
- width=image_width,
327
- height=image_height,
328
- )
329
-
330
-
331
-class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
332
-
316
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
317
return {"image": None}
318
319
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
337
- target_width, target_height = self._get_image_size_with_most_features()
320
+ target_width, target_height = self.get_image_size_with_most_features()
321
339
- max_image_tokens = self._get_num_image_tokens(
322
+ max_image_tokens = self.get_num_image_tokens(
323
image_width=target_width,
324
image_height=target_height,
325
+ processor=None,
326
)
327
328
return {"image": max_image_tokens}
329
346
- def _get_image_size_with_most_features(self) -> ImageSize:
330
+ def get_num_image_tokens(
331
+ self,
332
+ *,
333
+ image_width: int,
334
+ image_height: int,
335
+ processor: Optional[ProcessorMixin],
336
+ ) -> int:
337
+ if processor is None:
338
+ processor = self.get_hf_processor()
339
+
340
+ return processor.calc_num_image_tokens_from_image_size( # type: ignore
341
+ width=image_width,
342
+ height=image_height,
343
+ )
344
+
345
+ def get_image_size_with_most_features(self) -> ImageSize:
346
# Result in the max possible feature size (h:w = 16:1)
347
return ImageSize(height=8000, width=50)
348
349
+
350
+class Phi3VDummyInputsBuilder(BaseDummyInputsBuilder[Phi3VProcessingInfo]):
351
+
352
def get_dummy_processor_inputs(
353
self,
354
seq_len: int,
@@ -354,7 +356,8 @@ class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
356
) -> ProcessorInputs:
357
num_images = mm_counts.get("image", 0)
358
357
- target_width, target_height = self._get_image_size_with_most_features()
359
+ target_width, target_height = \
360
+ self.info.get_image_size_with_most_features()
361
362
mm_data = {
363
"image":
@@ -363,7 +366,7 @@ class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
366
num_images=num_images)
367
}
368
366
- hf_processor = self._get_hf_processor()
369
+ hf_processor = self.info.get_hf_processor()
370
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
371
372
return ProcessorInputs(
@@ -372,10 +375,7 @@ class Phi3VProfilingInfo(Phi3VProcessingMixin, BaseProfilingInfo):
375
)
376
377
375
-class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
376
-
377
- def _get_profiling_info(self) -> BaseProfilingInfo:
378
- return Phi3VProfilingInfo(self.ctx)
378
+class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
379
380
def _call_hf_processor(
381
self,
@@ -416,10 +416,10 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
416
hf_processor_mm_kwargs: Mapping[str, Any],
417
out_mm_kwargs: MultiModalKwargs,
418
) -> list[PromptReplacement]:
419
- hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
419
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
420
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
421
422
- tokenizer = self._get_tokenizer()
422
+ tokenizer = self.info.get_tokenizer()
423
bos_token_id = tokenizer.bos_token_id
424
assert isinstance(bos_token_id, int)
425
@@ -431,9 +431,10 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
431
num_image_tokens = images.get_feature_size(item_idx)
432
else:
433
image_size = images.get_image_size(item_idx)
434
- num_image_tokens = self._get_num_image_tokens(
434
+ num_image_tokens = self.info.get_num_image_tokens(
435
image_width=image_size.width,
436
image_height=image_size.height,
437
+ processor=hf_processor,
438
)
439
440
return [_IMAGE_TOKEN_ID] * num_image_tokens + [bos_token_id]
@@ -451,9 +452,9 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
452
def _apply_prompt_replacements(
453
self,
454
token_ids: list[int],
454
- mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
455
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
456
mm_item_counts: Mapping[str, int],
456
- ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
457
+ ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
458
token_ids, text, placeholders = super()._apply_prompt_replacements(
459
token_ids=token_ids,
460
mm_prompt_repls=mm_prompt_repls,
@@ -466,7 +467,7 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
467
token_ids = [token_ids[0], *token_ids[2:]]
468
placeholders = {
469
modality: [
469
- _PlaceholderInfo(
470
+ PlaceholderInfo(
471
modality=p.modality,
472
item_idx=p.item_idx,
473
start_idx=p.start_idx - 1,
@@ -499,7 +500,9 @@ class Phi3VMultiModalProcessor(Phi3VProcessingMixin, BaseMultiModalProcessor):
500
return result
501
502
502
-@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
503
+@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor,
504
+ info=Phi3VProcessingInfo,
505
+ dummy_inputs=Phi3VDummyInputsBuilder)
506
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
507
hf_to_vllm_mapper = WeightsMapper(
508
orig_to_new_prefix={
vllm/model_executor/models/qwen2_audio.py
+25
-24
@@ -38,11 +38,11 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
39
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
40
NestedTensors)
41
-from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataParser
41
+from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
42
+ MultiModalDataParser)
43
from vllm.multimodal.processing import (BaseMultiModalProcessor,
43
- MultiModalDataItems, ProcessingMixin,
44
- PromptReplacement)
45
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
44
+ BaseProcessingInfo, PromptReplacement)
45
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46
from vllm.sequence import IntermediateTensors
47
48
from .interfaces import SupportsMultiModal, SupportsPP
@@ -80,12 +80,12 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
80
return feat_lengths, output_lengths
81
82
83
-class Qwen2AudioProcessingMixin(ProcessingMixin):
83
+class Qwen2AudioProcessingInfo(BaseProcessingInfo):
84
85
- def _get_hf_config(self):
85
+ def get_hf_config(self):
86
return self.ctx.get_hf_config(Qwen2AudioConfig)
87
88
- def _get_hf_processor(
88
+ def get_hf_processor(
89
self,
90
*,
91
# Ignored in initialization
@@ -93,36 +93,37 @@ class Qwen2AudioProcessingMixin(ProcessingMixin):
93
) -> Qwen2AudioProcessor:
94
return self.ctx.get_hf_processor(Qwen2AudioProcessor)
95
96
- def _get_feature_extractor(
96
+ def get_feature_extractor(
97
self,
98
*,
99
# Ignored in initialization
100
sampling_rate: Optional[int] = None,
101
) -> WhisperFeatureExtractor:
102
- hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
102
+ hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
103
feature_extractor = hf_processor.feature_extractor # type: ignore
104
assert isinstance(feature_extractor, WhisperFeatureExtractor)
105
return feature_extractor
106
107
-
108
-class Qwen2AudioProfilingInfo(Qwen2AudioProcessingMixin, BaseProfilingInfo):
109
-
107
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
108
return {"audio": None}
109
110
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
114
- hf_config = self._get_hf_config()
111
+ hf_config = self.get_hf_config()
112
max_source_positions = hf_config.audio_config.max_source_positions
113
max_output_lengths = (max_source_positions - 2) // 2 + 1
114
115
return {"audio": max_output_lengths}
116
117
+
118
+class Qwen2AudioDummyInputsBuilder(
119
+ BaseDummyInputsBuilder[Qwen2AudioProcessingInfo]):
120
+
121
def get_dummy_processor_inputs(
122
self,
123
seq_len: int,
124
mm_counts: Mapping[str, int],
125
) -> ProcessorInputs:
125
- feature_extractor = self._get_feature_extractor()
126
+ feature_extractor = self.info.get_feature_extractor()
127
128
sampling_rate = feature_extractor.sampling_rate
129
audio_len = feature_extractor.chunk_length * sampling_rate
@@ -139,14 +140,11 @@ class Qwen2AudioProfilingInfo(Qwen2AudioProcessingMixin, BaseProfilingInfo):
140
)
141
142
142
-class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
143
- BaseMultiModalProcessor):
144
-
145
- def _get_profiling_info(self) -> BaseProfilingInfo:
146
- return Qwen2AudioProfilingInfo(self.ctx)
143
+class Qwen2AudioMultiModalProcessor(
144
+ BaseMultiModalProcessor[Qwen2AudioProcessingInfo]):
145
146
def _get_data_parser(self) -> MultiModalDataParser:
149
- feature_extractor = self._get_feature_extractor()
147
+ feature_extractor = self.info.get_feature_extractor()
148
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
149
150
def _call_hf_processor(
@@ -161,7 +159,7 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
159
if audios:
160
mm_data["audios"] = audios
161
164
- feature_extractor = self._get_feature_extractor(**mm_kwargs)
162
+ feature_extractor = self.info.get_feature_extractor(**mm_kwargs)
163
mm_kwargs = dict(
164
**mm_kwargs,
165
sampling_rate=feature_extractor.sampling_rate,
@@ -194,7 +192,7 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
192
hf_processor_mm_kwargs: Mapping[str, object],
193
out_mm_kwargs: MultiModalKwargs,
194
) -> list[PromptReplacement]:
197
- hf_config = self._get_hf_config()
195
+ hf_config = self.info.get_hf_config()
196
placeholder = hf_config.audio_token_index
197
198
feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
@@ -234,10 +232,13 @@ class Qwen2AudioMultiModalProcessor(Qwen2AudioProcessingMixin,
232
# has already performed processing for multi-audio input when the input
233
# audios are short (the corresponding placeholders may take up fewer
234
# tokens than the number of audio items)
237
- return not hasattr(self._get_hf_processor(), "audio_token")
235
+ return not hasattr(self.info.get_hf_processor(), "audio_token")
236
237
240
-@MULTIMODAL_REGISTRY.register_processor(Qwen2AudioMultiModalProcessor)
238
+@MULTIMODAL_REGISTRY.register_processor(
239
+ Qwen2AudioMultiModalProcessor,
240
+ info=Qwen2AudioProcessingInfo,
241
+ dummy_inputs=Qwen2AudioDummyInputsBuilder)
242
class Qwen2AudioForConditionalGeneration(nn.Module, SupportsMultiModal,
243
SupportsPP):
244
vllm/model_executor/models/qwen2_vl.py
+63
-50
@@ -57,11 +57,10 @@ from vllm.multimodal.inputs import (ImageItem, ModalityData,
57
MultiModalFieldConfig, MultiModalKwargs,
58
NestedTensors, VideoItem)
59
from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60
- MultiModalDataParser)
60
+ MultiModalDataItems, MultiModalDataParser)
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
62
- MultiModalDataItems, ProcessingMixin,
63
- PromptReplacement)
64
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
62
+ BaseProcessingInfo, PromptReplacement)
63
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
64
from vllm.platforms import _Backend
65
from vllm.sequence import IntermediateTensors
66
from vllm.transformers_utils.config import uses_mrope
@@ -709,12 +708,12 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
708
return super()._parse_video_data(data)
709
710
712
-class Qwen2VLProcessingMixin(ProcessingMixin):
711
+class Qwen2VLProcessingInfo(BaseProcessingInfo):
712
714
- def _get_hf_config(self):
713
+ def get_hf_config(self):
714
return self.ctx.get_hf_config(Qwen2VLConfig)
715
717
- def _get_hf_processor(
716
+ def get_hf_processor(
717
self,
718
*,
719
min_pixels: Optional[int] = None,
@@ -736,18 +735,27 @@ class Qwen2VLProcessingMixin(ProcessingMixin):
735
736
return hf_processor
737
739
- def _get_image_processor(
738
+ def get_image_processor(
739
self,
740
*,
741
min_pixels: Optional[int] = None,
742
max_pixels: Optional[int] = None,
743
):
745
- hf_processor = self._get_hf_processor(min_pixels=min_pixels,
746
- max_pixels=max_pixels)
744
+ hf_processor = self.get_hf_processor(min_pixels=min_pixels,
745
+ max_pixels=max_pixels)
746
image_processor = hf_processor.image_processor # type: ignore
747
assert isinstance(image_processor, Qwen2VLImageProcessor)
748
return image_processor
749
750
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
751
+ return {"image": None, "video": None}
752
+
753
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
754
+ return {
755
+ "image": self.get_max_image_tokens(),
756
+ "video": self.get_max_video_tokens(seq_len),
757
+ }
758
+
759
def _get_vision_info(
760
self,
761
*,
@@ -755,15 +763,17 @@ class Qwen2VLProcessingMixin(ProcessingMixin):
763
image_height: int,
764
num_frames: int = 1,
765
do_resize: bool = True,
766
+ image_processor: Optional[Qwen2VLImageProcessor],
767
) -> tuple[ImageSize, int]:
759
- hf_config = self._get_hf_config()
768
+ if image_processor is None:
769
+ image_processor = self.get_image_processor()
770
+
771
+ hf_config = self.get_hf_config()
772
vision_config = hf_config.vision_config
773
patch_size = vision_config.patch_size
774
merge_size = vision_config.spatial_merge_size
775
temporal_patch_size = vision_config.temporal_patch_size
776
765
- image_processor = self._get_image_processor()
766
-
777
if do_resize:
778
resized_height, resized_width = smart_resize(
779
height=image_height,
@@ -787,70 +797,65 @@ class Qwen2VLProcessingMixin(ProcessingMixin):
797
798
return preprocessed_size, num_vision_tokens
799
790
- def _get_num_image_tokens(
800
+ def get_num_image_tokens(
801
self,
802
*,
803
image_width: int,
804
image_height: int,
805
+ image_processor: Optional[Qwen2VLImageProcessor],
806
) -> int:
807
_, num_image_tokens = self._get_vision_info(
808
image_width=image_width,
809
image_height=image_height,
810
+ image_processor=image_processor,
811
)
812
return num_image_tokens
813
802
- def _get_num_video_tokens(
814
+ def get_num_video_tokens(
815
self,
816
*,
817
image_width: int,
818
image_height: int,
819
num_frames: int,
820
+ image_processor: Optional[Qwen2VLImageProcessor],
821
) -> int:
822
_, num_video_tokens = self._get_vision_info(
823
image_width=image_width,
824
image_height=image_height,
825
num_frames=num_frames,
826
+ image_processor=image_processor,
827
)
828
return num_video_tokens
829
816
-
817
-class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
818
-
819
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
820
- return {"image": None, "video": None}
821
-
822
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
823
- return {
824
- "image": self._get_max_image_tokens(),
825
- "video": self._get_max_video_tokens(seq_len),
826
- }
827
-
828
- def _get_image_size_with_most_features(self) -> ImageSize:
830
+ def get_image_size_with_most_features(self) -> ImageSize:
831
max_image_size, _ = self._get_vision_info(
832
image_width=9999999,
833
image_height=9999999,
834
+ image_processor=None,
835
)
836
return max_image_size
837
835
- def _get_max_image_tokens(self) -> int:
836
- target_width, target_height = self._get_image_size_with_most_features()
838
+ def get_max_image_tokens(self) -> int:
839
+ target_width, target_height = self.get_image_size_with_most_features()
840
838
- return self._get_num_image_tokens(
841
+ return self.get_num_image_tokens(
842
image_width=target_width,
843
image_height=target_height,
844
+ image_processor=None,
845
)
846
847
def _get_max_video_frames(self, max_tokens: int) -> int:
844
- target_width, target_height = self._get_image_size_with_most_features()
848
+ target_width, target_height = self.get_image_size_with_most_features()
849
850
num_frames = 0
851
852
while True:
853
next_num_frames = num_frames + 1
850
- next_max_tokens = self._get_num_video_tokens(
854
+ next_max_tokens = self.get_num_video_tokens(
855
image_width=target_width,
856
image_height=target_height,
857
num_frames=next_num_frames,
858
+ image_processor=None,
859
)
860
861
if next_max_tokens > max_tokens:
@@ -860,12 +865,12 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
865
866
return num_frames
867
863
- def _get_dummy_num_frames(self, seq_len: int) -> int:
868
+ def get_num_frames_with_most_features(self, seq_len: int) -> int:
869
mm_config = self.ctx.get_mm_config()
870
max_images = mm_config.limit_per_prompt.get("image", 1)
871
max_videos = mm_config.limit_per_prompt.get("video", 1)
872
868
- max_image_tokens = self._get_max_image_tokens() * max_images
873
+ max_image_tokens = self.get_max_image_tokens() * max_images
874
max_total_frames = self._get_max_video_frames(seq_len -
875
max_image_tokens)
876
@@ -877,15 +882,19 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
882
883
return num_frames
884
880
- def _get_max_video_tokens(self, seq_len: int) -> int:
881
- target_width, target_height = self._get_image_size_with_most_features()
885
+ def get_max_video_tokens(self, seq_len: int) -> int:
886
+ target_width, target_height = self.get_image_size_with_most_features()
887
883
- return self._get_num_video_tokens(
888
+ return self.get_num_video_tokens(
889
image_width=target_width,
890
image_height=target_height,
886
- num_frames=self._get_dummy_num_frames(seq_len),
891
+ num_frames=self.get_num_frames_with_most_features(seq_len),
892
+ image_processor=None,
893
)
894
895
+
896
+class Qwen2VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen2VLProcessingInfo]):
897
+
898
def get_dummy_processor_inputs(
899
self,
900
seq_len: int,
@@ -894,10 +903,14 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
903
num_images = mm_counts.get("image", 0)
904
num_videos = mm_counts.get("video", 0)
905
897
- hf_processor = self._get_hf_processor()
906
+ hf_processor = self.info.get_hf_processor()
907
image_token: str = hf_processor.image_token
908
video_token: str = hf_processor.video_token
900
- target_width, target_height = self._get_image_size_with_most_features()
909
+
910
+ target_width, target_height = \
911
+ self.info.get_image_size_with_most_features()
912
+ target_num_frames = \
913
+ self.info.get_num_frames_with_most_features(seq_len)
914
915
mm_data = {
916
"image":
@@ -908,7 +921,7 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
921
self._get_dummy_videos(
922
width=target_width,
923
height=target_height,
911
- num_frames=self._get_dummy_num_frames(seq_len),
924
+ num_frames=target_num_frames,
925
num_videos=num_videos,
926
)
927
}
@@ -919,11 +932,8 @@ class Qwen2VLProfilingInfo(Qwen2VLProcessingMixin, BaseProfilingInfo):
932
)
933
934
922
-class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
923
- BaseMultiModalProcessor):
924
-
925
- def _get_profiling_info(self) -> BaseProfilingInfo:
926
- return Qwen2VLProfilingInfo(self.ctx)
935
+class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
936
+ ):
937
938
def _get_data_parser(self) -> MultiModalDataParser:
939
return Qwen2MultiModalDataParser()
@@ -934,8 +944,9 @@ class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
944
hf_processor_mm_kwargs: Mapping[str, Any],
945
out_mm_kwargs: MultiModalKwargs,
946
) -> list[PromptReplacement]:
937
- hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
938
- image_processor = self._get_image_processor(**hf_processor_mm_kwargs)
947
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
948
+ image_processor = self.info.get_image_processor(
949
+ **hf_processor_mm_kwargs)
950
951
# NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
952
# image_token and video_token registered
@@ -991,7 +1002,9 @@ class Qwen2VLMultiModalProcessor(Qwen2VLProcessingMixin,
1002
)
1003
1004
994
-@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor)
1005
+@MULTIMODAL_REGISTRY.register_processor(Qwen2VLMultiModalProcessor,
1006
+ info=Qwen2VLProcessingInfo,
1007
+ dummy_inputs=Qwen2VLDummyInputsBuilder)
1008
class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1009
SupportsLoRA, SupportsPP):
1010
packed_modules_mapping = {
vllm/model_executor/models/ultravox.py
+23
-23
@@ -24,11 +24,10 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
24
from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
26
NestedTensors)
27
-from vllm.multimodal.parse import MultiModalDataParser
27
+from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
28
from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
- MultiModalDataItems, ProcessingMixin,
30
- PromptReplacement)
31
-from vllm.multimodal.profiling import BaseProfilingInfo, ProcessorInputs
29
+ BaseProcessingInfo, PromptReplacement)
30
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31
from vllm.sequence import IntermediateTensors
32
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
33
@@ -59,9 +58,9 @@ UltravoxAudioInputs = Union[UltravoxAudioFeatureInputs,
58
UltravoxAudioEmbeddingInputs]
59
60
62
-class UltravoxProcessingMixin(ProcessingMixin):
61
+class UltravoxProcessingInfo(BaseProcessingInfo):
62
64
- def _get_hf_processor(
63
+ def get_hf_processor(
64
self,
65
*,
66
# Ignored in initialization
@@ -76,37 +75,38 @@ class UltravoxProcessingMixin(ProcessingMixin):
75
hf_processor.audio_token_replacement = _AUDIO_PLACEHOLDER_OVERRIDE
76
return hf_processor
77
79
- def _get_feature_extractor(
78
+ def get_feature_extractor(
79
self,
80
*,
81
# Ignored in initialization
82
sampling_rate: Optional[int] = None,
83
) -> WhisperFeatureExtractor:
85
- hf_processor = self._get_hf_processor(sampling_rate=sampling_rate)
84
+ hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
85
audio_processor = hf_processor.audio_processor # type: ignore
86
feature_extractor = audio_processor.feature_extractor # type: ignore
87
assert isinstance(feature_extractor, WhisperFeatureExtractor)
88
return feature_extractor
89
91
-
92
-class UltravoxProfilingInfo(UltravoxProcessingMixin, BaseProfilingInfo):
93
-
90
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
91
return {"audio": None}
92
93
def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
98
- feature_extractor = self._get_feature_extractor()
94
+ feature_extractor = self.get_feature_extractor()
95
max_audio_tokens = math.ceil(feature_extractor.chunk_length *
96
_AUDIO_TOKENS_PER_SECOND)
97
98
return {"audio": max_audio_tokens}
99
100
+
101
+class UltravoxDummyInputsBuilder(BaseDummyInputsBuilder[UltravoxProcessingInfo]
102
+ ):
103
+
104
def get_dummy_processor_inputs(
105
self,
106
seq_len: int,
107
mm_counts: Mapping[str, int],
108
) -> ProcessorInputs:
109
- feature_extractor = self._get_feature_extractor()
109
+ feature_extractor = self.info.get_feature_extractor()
110
111
sampling_rate = feature_extractor.sampling_rate
112
audio_len = feature_extractor.chunk_length * sampling_rate
@@ -123,14 +123,11 @@ class UltravoxProfilingInfo(UltravoxProcessingMixin, BaseProfilingInfo):
123
)
124
125
126
-class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
127
- BaseMultiModalProcessor):
128
-
129
- def _get_profiling_info(self) -> BaseProfilingInfo:
130
- return UltravoxProfilingInfo(self.ctx)
126
+class UltravoxMultiModalProcessor(
127
+ BaseMultiModalProcessor[UltravoxProcessingInfo]):
128
129
def _get_data_parser(self) -> MultiModalDataParser:
133
- feature_extractor = self._get_feature_extractor()
130
+ feature_extractor = self.info.get_feature_extractor()
131
return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
132
133
def _call_hf_processor(
@@ -141,7 +138,7 @@ class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
138
) -> BatchFeature:
139
# Text-only input not supported in composite processor
140
if not mm_data:
144
- tokenizer = self._get_tokenizer()
141
+ tokenizer = self.info.get_tokenizer()
142
143
prompt_ids = tokenizer.encode(
144
prompt,
@@ -160,7 +157,7 @@ class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
157
mm_kwargs=mm_kwargs,
158
)
159
163
- feature_extractor = self._get_feature_extractor()
160
+ feature_extractor = self.info.get_feature_extractor()
161
mm_kwargs = dict(
162
**mm_kwargs,
163
sampling_rate=feature_extractor.sampling_rate,
@@ -208,7 +205,7 @@ class UltravoxMultiModalProcessor(UltravoxProcessingMixin,
205
hf_processor_mm_kwargs: Mapping[str, Any],
206
out_mm_kwargs: MultiModalKwargs,
207
) -> list[PromptReplacement]:
211
- hf_processor = self._get_hf_processor(**hf_processor_mm_kwargs)
208
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
209
placeholder = hf_processor.audio_token_replacement # type: ignore
210
211
def get_replacement_ultravox(item_idx: int):
@@ -342,7 +339,10 @@ class ModifiedWhisperEncoder(WhisperEncoder):
339
return hidden_states
340
341
345
-@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor)
342
+@MULTIMODAL_REGISTRY.register_processor(UltravoxMultiModalProcessor,
343
+ info=UltravoxProcessingInfo,
344
+ dummy_inputs=UltravoxDummyInputsBuilder
345
+ )
346
class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP):
347
348
hf_to_vllm_mapper = WeightsMapper(
vllm/multimodal/processing.py
+85
-134
@@ -4,12 +4,13 @@ from collections import defaultdict
4
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
5
from dataclasses import dataclass, field
6
from functools import lru_cache
7
-from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
7
+from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
8
+ TypeVar, Union)
9
10
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
11
11
-from vllm import envs
12
-from vllm.inputs import DummyData, InputProcessingContext
12
+import vllm.envs as envs
13
+from vllm.inputs import InputProcessingContext
14
from vllm.logger import init_logger
15
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
16
encode_tokens)
@@ -20,7 +21,9 @@ from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
21
MultiModalInputsV2, MultiModalKwargs,
22
MultiModalKwargsItem, PlaceholderRange)
23
from .parse import MultiModalDataItems, MultiModalDataParser
23
-from .profiling import BaseProfilingInfo
24
+
25
+if TYPE_CHECKING:
26
+ from .profiling import BaseDummyInputsBuilder
27
28
logger = init_logger(__name__)
29
@@ -46,8 +49,8 @@ class PromptReplacement:
49
if it does not depend on the input.
50
"""
51
49
- def bind(self, tokenizer: AnyTokenizer) -> "_BoundPromptReplacement":
50
- return _BoundPromptReplacement(
52
+ def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptReplacement":
53
+ return BoundPromptReplacement(
54
tokenizer=tokenizer,
55
modality=self.modality,
56
_target=self.target,
@@ -128,7 +131,7 @@ class _BoundPromptSequence:
131
132
133
@dataclass
131
-class _BoundPromptReplacement:
134
+class BoundPromptReplacement:
135
tokenizer: AnyTokenizer = field(repr=False)
136
modality: str
137
@@ -207,7 +210,7 @@ def iter_token_matches(
210
211
@dataclass(repr=False)
212
class _PromptReplacementMatch(ABC):
210
- prompt_repl: _BoundPromptReplacement
213
+ prompt_repl: BoundPromptReplacement
214
215
@property
216
def modality(self) -> str:
@@ -255,7 +258,7 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch):
258
259
260
@dataclass
258
-class _PlaceholderInfo:
261
+class PlaceholderInfo:
262
modality: str
263
item_idx: int
264
start_idx: int
@@ -274,7 +277,7 @@ class _PlaceholderInfo:
277
278
def find_token_matches(
279
prompt: list[int],
277
- prompt_repls: Sequence[_BoundPromptReplacement],
280
+ prompt_repls: Sequence[BoundPromptReplacement],
281
) -> list[_PromptReplacementTokenMatch]:
282
"""Return each target of :code:`prompt_repls` found in :code:`prompt`."""
283
return [
@@ -286,7 +289,7 @@ def find_token_matches(
289
290
def find_text_matches(
291
prompt: str,
289
- prompt_repls: Sequence[_BoundPromptReplacement],
292
+ prompt_repls: Sequence[BoundPromptReplacement],
293
) -> list[_PromptReplacementTextMatch]:
294
"""Return each target of :code:`prompt_repls` found in :code:`prompt`."""
295
return [
@@ -390,9 +393,9 @@ def replace_text_matches(
393
def _iter_modality_placeholders(
394
prompt: list[int],
395
modality: str,
393
- modality_repls: Sequence[_BoundPromptReplacement],
396
+ modality_repls: Sequence[BoundPromptReplacement],
397
modal_item_count: int,
395
-) -> Iterable[_PlaceholderInfo]:
398
+) -> Iterable[PlaceholderInfo]:
399
if modal_item_count == 0:
400
return
401
@@ -413,7 +416,7 @@ def _iter_modality_placeholders(
416
continue
417
418
if prompt[start_idx:end_idx] == repl_tokens:
416
- yield _PlaceholderInfo(
419
+ yield PlaceholderInfo(
420
modality=modality,
421
item_idx=item_idx,
422
start_idx=start_idx,
@@ -434,10 +437,10 @@ def _iter_modality_placeholders(
437
438
439
def _iter_placeholders(
437
- mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
440
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
441
prompt: list[int],
442
mm_item_counts: Mapping[str, int],
440
-) -> Iterable[_PlaceholderInfo]:
443
+) -> Iterable[PlaceholderInfo]:
444
"""
445
For each modality, yield each set of placeholder tokens found in
446
:code:`prompt`.
@@ -455,10 +458,10 @@ def _iter_placeholders(
458
459
460
def find_mm_placeholders(
458
- mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
461
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
462
prompt: list[int],
463
mm_item_counts: Mapping[str, int],
461
-) -> Mapping[str, list[_PlaceholderInfo]]:
464
+) -> Mapping[str, list[PlaceholderInfo]]:
465
it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
466
return dict(full_groupby_modality(it))
467
@@ -524,29 +527,59 @@ class ProcessingCache:
527
self._cache.put(cache_key, output_kwargs)
528
529
527
-class ProcessingMixin:
528
- """
529
- Contains helper functions to perform processing.
530
+class BaseProcessingInfo:
531
+ """Base class containing information to perform processing."""
532
531
- Not to be confused with :class:`transformers.ProcessorMixin`.
532
- """
533
- ctx: InputProcessingContext
533
+ def __init__(self, ctx: InputProcessingContext) -> None:
534
+ super().__init__()
535
535
- def _get_tokenizer(self) -> AnyTokenizer:
536
+ self.ctx = ctx
537
+
538
+ @property
539
+ def model_id(self) -> str:
540
+ return self.ctx.model_config.model
541
+
542
+ def get_tokenizer(self) -> AnyTokenizer:
543
return self.ctx.tokenizer
544
538
- def _get_hf_config(self) -> PretrainedConfig:
545
+ def get_hf_config(self) -> PretrainedConfig:
546
return self.ctx.get_hf_config()
547
541
- def _get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
548
+ def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
549
"""
550
Subclasses can override this method to handle
551
specific kwargs from model config or user inputs.
552
"""
553
return self.ctx.get_hf_processor(**kwargs)
554
555
+ @abstractmethod
556
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
557
+ """
558
+ Return the maximum supported number of items for each modality.
559
+
560
+ A value of `None` means unlimited number of items.
561
+
562
+ Omitting a modality from the returned dictionary means that
563
+ it is not supported at all.
564
+ """
565
+ raise NotImplementedError
566
+
567
+ @abstractmethod
568
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
569
+ """
570
+ Get the maximum possible number of tokens per data item
571
+ for each modality.
572
+
573
+ The dictionary returned by this method should have the same
574
+ keys as that returned by :meth:`get_supported_mm_limits`.
575
+ """
576
+ raise NotImplementedError
577
+
578
+
579
+_I = TypeVar("_I", bound=BaseProcessingInfo)
580
549
-class BaseMultiModalProcessor(ProcessingMixin, ABC):
581
+
582
+class BaseMultiModalProcessor(ABC, Generic[_I]):
583
"""
584
Abstract base class to process multi-modal inputs to be used in vLLM.
585
@@ -554,18 +587,19 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
587
"""
588
589
def __init__(self,
557
- ctx: InputProcessingContext,
590
+ info: _I,
591
+ dummy_inputs: "BaseDummyInputsBuilder[_I]",
592
*,
593
cache: Optional[ProcessingCache] = None,
594
enable_sanity_checks: bool = True) -> None:
595
super().__init__()
596
563
- self.ctx = ctx
597
+ self.info = info
598
+ self.dummy_inputs = dummy_inputs
599
self.cache = cache
600
self.enable_sanity_checks = enable_sanity_checks
601
602
self.data_parser = self._get_data_parser()
568
- self.profiling_info = self._get_profiling_info()
603
604
def __call__(
605
self,
@@ -585,13 +619,6 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
619
"""
620
return MultiModalDataParser()
621
588
- def _get_profiling_info(self) -> BaseProfilingInfo:
589
- """
590
- Get the profiling information to find the worst-case memory usage of
591
- the model.
592
- """
593
- raise NotImplementedError
594
-
622
def _to_mm_items(
623
self,
624
mm_data: MultiModalDataDict,
@@ -602,7 +629,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
629
"""
630
mm_items = self.data_parser.parse_mm_data(mm_data)
631
605
- mm_limits = self.ctx.get_mm_config().limit_per_prompt
632
+ mm_limits = self.info.ctx.get_mm_config().limit_per_prompt
633
for modality, items in mm_items.items():
634
limit = mm_limits.get(modality, 1)
635
if len(items) > limit:
@@ -646,19 +673,19 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
673
674
def _find_mm_placeholders(
675
self,
649
- mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
676
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
677
new_token_ids: list[int],
678
mm_item_counts: Mapping[str, int],
652
- ) -> Mapping[str, list[_PlaceholderInfo]]:
679
+ ) -> Mapping[str, list[PlaceholderInfo]]:
680
return find_mm_placeholders(mm_prompt_repls, new_token_ids,
681
mm_item_counts)
682
683
def _get_hf_mm_data(
684
self,
685
mm_items: MultiModalDataItems,
659
- ) -> tuple[dict[str, Any], dict[str, Any]]:
660
- processor_data = dict[str, Any]()
661
- passthrough_data = dict[str, Any]()
686
+ ) -> tuple[Mapping[str, object], Mapping[str, object]]:
687
+ processor_data = dict[str, object]()
688
+ passthrough_data = dict[str, object]()
689
690
for items in mm_items.values():
691
processor_data.update(items.get_processor_data())
@@ -678,8 +705,8 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
705
Call the HF processor on the prompt text and
706
associated multi-modal data.
707
"""
681
- return self.ctx.call_hf_processor(
682
- self._get_hf_processor(**mm_kwargs),
708
+ return self.info.ctx.call_hf_processor(
709
+ self.info.get_hf_processor(**mm_kwargs),
710
dict(text=prompt, **mm_data),
711
mm_kwargs,
712
)
@@ -738,8 +765,8 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
765
766
# Some HF processors (e.g. Qwen2-VL) expect corresponding
767
# multi-modal tokens to be in the prompt text
741
- dummy_inputs = self.profiling_info.get_dummy_processor_inputs(
742
- self.ctx.model_config.max_model_len,
768
+ dummy_inputs = self.dummy_inputs.get_dummy_processor_inputs(
769
+ self.info.ctx.model_config.max_model_len,
770
mm_missing_counts,
771
)
772
@@ -762,7 +789,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
789
caching the results and reusing cached results.
790
"""
791
cache = self.cache
765
- model_id = self.ctx.model_config.model
792
+ model_id = self.info.model_id
793
794
_, passthrough_data = self._get_hf_mm_data(mm_data_items)
795
if cache is None or passthrough_data:
@@ -838,8 +865,8 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
865
def _bind_and_group_repls(
866
self,
867
prompt_repls: list[PromptReplacement],
841
- ) -> dict[str, list[_BoundPromptReplacement]]:
842
- tokenizer = self._get_tokenizer()
868
+ ) -> dict[str, list[BoundPromptReplacement]]:
869
+ tokenizer = self.info.get_tokenizer()
870
871
it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
872
return dict(full_groupby_modality(it))
@@ -859,10 +886,10 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
886
def _apply_prompt_replacements(
887
self,
888
token_ids: list[int],
862
- mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
889
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
890
mm_item_counts: Mapping[str, int],
864
- ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
865
- tokenizer = self._get_tokenizer()
891
+ ) -> tuple[list[int], str, Mapping[str, list[PlaceholderInfo]]]:
892
+ tokenizer = self.info.get_tokenizer()
893
894
mm_token_matches = {
895
modality: find_token_matches(token_ids, prompt_repls)
@@ -950,7 +977,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
977
978
def _validate_mm_placeholders(
979
self,
953
- mm_placeholders: Mapping[str, list[_PlaceholderInfo]],
980
+ mm_placeholders: Mapping[str, list[PlaceholderInfo]],
981
mm_item_counts: Mapping[str, int],
982
*,
983
allow_missing: bool = False,
@@ -1001,7 +1028,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1028
# instead of rehashing.
1029
1030
if envs.VLLM_USE_V1:
1004
- model_id = self.ctx.model_config.model
1031
+ model_id = self.info.model_id
1032
mm_hashes = {
1033
modality: [
1034
MultiModalHasher.hash_kwargs(model_id=model_id,
@@ -1046,7 +1073,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1073
allow_missing=True,
1074
)
1075
1049
- mm_missing_repls = dict[str, list[_BoundPromptReplacement]]()
1076
+ mm_missing_repls = dict[str, list[BoundPromptReplacement]]()
1077
for modality, missing_repl_count in mm_missing_repl_counts.items():
1078
if missing_repl_count == 0:
1079
mm_missing_repls[modality] = []
@@ -1059,7 +1086,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1086
# If HF processor already inserts placeholder tokens,
1087
# there is no need for us to insert them
1088
if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1062
- tokenizer = self._get_tokenizer()
1089
+ tokenizer = self.info.get_tokenizer()
1090
prompt_text = decode_tokens(tokenizer, prompt_ids)
1091
mm_placeholders = hf_mm_placeholders
1092
else:
@@ -1090,79 +1117,3 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
1117
mm_hashes=mm_hashes,
1118
mm_placeholders=mm_placeholder_ranges,
1119
)
1093
-
1094
- def _get_dummy_mm_inputs(
1095
- self,
1096
- seq_len: int,
1097
- mm_counts: Mapping[str, int],
1098
- ) -> MultiModalInputsV2:
1099
- profiling = self.profiling_info
1100
- processor_inputs = profiling.get_dummy_processor_inputs(
1101
- seq_len, mm_counts)
1102
-
1103
- return self.apply(
1104
- prompt_text=processor_inputs.prompt_text,
1105
- mm_data=processor_inputs.mm_data,
1106
- hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1107
- )
1108
-
1109
- def get_dummy_data(self, seq_len: int) -> DummyData:
1110
- # Avoid circular import
1111
- from vllm.sequence import SequenceData
1112
-
1113
- profiling = self.profiling_info
1114
- mm_counts = profiling.get_mm_limits()
1115
- mm_max_tokens_per_item = profiling.get_mm_max_tokens_per_item(seq_len)
1116
- if mm_counts.keys() != mm_max_tokens_per_item.keys():
1117
- raise AssertionError(
1118
- "The keys returned by `get_supported_mm_limits`"
1119
- f"({set(mm_counts.keys())}) should be the same as those "
1120
- "returned by `get_mm_max_tokens_per_item` "
1121
- f"({set(mm_max_tokens_per_item.keys())})")
1122
-
1123
- mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
1124
- prompt_token_ids = mm_inputs["prompt_token_ids"]
1125
- placeholders_by_modality = mm_inputs["mm_placeholders"]
1126
-
1127
- total_placeholders_by_modality = {
1128
- modality: sum(item["length"] for item in placeholders)
1129
- for modality, placeholders in placeholders_by_modality.items()
1130
- }
1131
- expected_placeholders_by_modality = {
1132
- modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
1133
- for modality in placeholders_by_modality
1134
- }
1135
- if total_placeholders_by_modality != expected_placeholders_by_modality:
1136
- raise AssertionError(
1137
- f"The processed dummy data has a total of "
1138
- f"{total_placeholders_by_modality} placeholder tokens, which "
1139
- f"is not the expected {expected_placeholders_by_modality} "
1140
- "tokens.")
1141
-
1142
- total_len = len(prompt_token_ids)
1143
-
1144
- # V0 does not support chunked prefill.
1145
- if total_len > seq_len and not envs.VLLM_USE_V1:
1146
- logger.warning(
1147
- "The context length (%d) of the model is too short "
1148
- "to hold the multi-modal embeddings in the worst case "
1149
- "(%d tokens in total, out of which %s are reserved for "
1150
- "multi-modal embeddings). This may cause certain multi-modal "
1151
- "inputs to fail during inference, even when the input text is "
1152
- "short. To avoid this, you should increase `max_model_len`, "
1153
- "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
1154
- total_len, total_placeholders_by_modality)
1155
-
1156
- return DummyData(
1157
- seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
1158
- multi_modal_data=None,
1159
- multi_modal_placeholders=None,
1160
- )
1161
-
1162
- prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
1163
-
1164
- return DummyData(
1165
- seq_data=SequenceData.from_seqs(prompt_token_ids),
1166
- multi_modal_data=mm_inputs["mm_kwargs"],
1167
- multi_modal_placeholders=placeholders_by_modality,
1168
- )
vllm/multimodal/profiling.py
+117
-35
@@ -1,16 +1,18 @@
1
from abc import ABC, abstractmethod
2
from collections.abc import Mapping
3
from dataclasses import dataclass, field
4
-from typing import Optional
4
+from typing import Generic, TypeVar
5
6
import numpy as np
7
import numpy.typing as npt
8
from PIL import Image
9
10
-from vllm.inputs import InputProcessingContext
10
+import vllm.envs as envs
11
+from vllm.inputs import DummyData
12
from vllm.logger import init_logger
13
13
-from .inputs import MultiModalDataDict
14
+from .inputs import MultiModalDataDict, MultiModalInputsV2
15
+from .processing import BaseMultiModalProcessor, BaseProcessingInfo
16
17
logger = init_logger(__name__)
18
@@ -23,39 +25,19 @@ class ProcessorInputs:
25
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
26
27
26
-class BaseProfilingInfo(ABC):
28
+_I = TypeVar("_I", bound=BaseProcessingInfo)
29
+
30
+
31
+class BaseDummyInputsBuilder(ABC, Generic[_I]):
32
"""
28
- Abstract base class that provides the information necessary to profile
33
+ Abstract base class that constructs the dummy data to profile
34
multi-modal models.
35
"""
36
32
- def __init__(self, ctx: InputProcessingContext) -> None:
37
+ def __init__(self, info: _I) -> None:
38
super().__init__()
39
35
- self.ctx = ctx
36
-
37
- @abstractmethod
38
- def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
39
- """
40
- Return the maximum supported number of items for each modality.
41
-
42
- A value of `None` means unlimited number of items.
43
-
44
- Omitting a modality from the returned dictionary means that
45
- it is not supported at all.
46
- """
47
- raise NotImplementedError
48
-
49
- @abstractmethod
50
- def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
51
- """
52
- Get the maximum possible number of tokens per data item
53
- for each modality.
54
-
55
- The dictionary returned by this method should have the same
56
- keys as that returned by :meth:`get_supported_mm_limits`.
57
- """
58
- raise NotImplementedError
40
+ self.info = info
41
42
@abstractmethod
43
def get_dummy_processor_inputs(
@@ -64,8 +46,8 @@ class BaseProfilingInfo(ABC):
46
mm_counts: Mapping[str, int],
47
) -> ProcessorInputs:
48
"""
67
- Build the multi-modal portion of the input which, after processing,
68
- results in `mm_max_tokens` in :meth:`get_mm_max_tokens_per_item`.
49
+ Build the input which, after processing, results in
50
+ `self.info.get_mm_max_tokens_per_item()` placeholder tokens.
51
"""
52
raise NotImplementedError
53
@@ -99,11 +81,33 @@ class BaseProfilingInfo(ABC):
81
video = np.zeros((num_frames, width, height, 3))
82
return [video] * num_videos
83
102
- def get_mm_limits(self) -> Mapping[str, int]:
103
- mm_config = self.ctx.get_mm_config()
84
+
85
+class MultiModalProfiler(Generic[_I]):
86
+ """
87
+ Contains code for running memory profiling for multi-modal models.
88
+ """
89
+
90
+ def __init__(
91
+ self,
92
+ processor: BaseMultiModalProcessor[_I],
93
+ ) -> None:
94
+ super().__init__()
95
+
96
+ self.processor = processor
97
+
98
+ @property
99
+ def processing_info(self) -> BaseProcessingInfo:
100
+ return self.processor.info
101
+
102
+ @property
103
+ def dummy_inputs(self) -> BaseDummyInputsBuilder[_I]:
104
+ return self.processor.dummy_inputs
105
+
106
+ def _get_mm_limits(self) -> Mapping[str, int]:
107
+ mm_config = self.processing_info.ctx.get_mm_config()
108
mm_limit_per_prompt = mm_config.limit_per_prompt
109
106
- supported_mm_limits = self.get_supported_mm_limits()
110
+ supported_mm_limits = self.processing_info.get_supported_mm_limits()
111
112
mm_limits = {
113
modality: mm_limit_per_prompt.get(modality, 1)
@@ -119,3 +123,81 @@ class BaseProfilingInfo(ABC):
123
f"at most {supported_limit} {modality} items.")
124
125
return mm_limits
126
+
127
+ def _get_dummy_mm_inputs(
128
+ self,
129
+ seq_len: int,
130
+ mm_counts: Mapping[str, int],
131
+ ) -> MultiModalInputsV2:
132
+ factory = self.dummy_inputs
133
+ processor_inputs = factory.get_dummy_processor_inputs(
134
+ seq_len, mm_counts)
135
+
136
+ return self.processor.apply(
137
+ prompt_text=processor_inputs.prompt_text,
138
+ mm_data=processor_inputs.mm_data,
139
+ hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
140
+ )
141
+
142
+ def get_dummy_data(self, seq_len: int) -> DummyData:
143
+ # Avoid circular import
144
+ from vllm.sequence import SequenceData
145
+
146
+ mm_counts = self._get_mm_limits()
147
+
148
+ info = self.processing_info
149
+ mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(seq_len)
150
+
151
+ if mm_counts.keys() != mm_max_tokens_per_item.keys():
152
+ raise AssertionError(
153
+ "The keys returned by `get_supported_mm_limits`"
154
+ f"({set(mm_counts.keys())}) should be the same as those "
155
+ "returned by `get_mm_max_tokens_per_item` "
156
+ f"({set(mm_max_tokens_per_item.keys())})")
157
+
158
+ mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
159
+ prompt_token_ids = mm_inputs["prompt_token_ids"]
160
+ placeholders_by_modality = mm_inputs["mm_placeholders"]
161
+
162
+ total_placeholders_by_modality = {
163
+ modality: sum(item["length"] for item in placeholders)
164
+ for modality, placeholders in placeholders_by_modality.items()
165
+ }
166
+ expected_placeholders_by_modality = {
167
+ modality: mm_max_tokens_per_item[modality] * mm_counts[modality]
168
+ for modality in placeholders_by_modality
169
+ }
170
+ if total_placeholders_by_modality != expected_placeholders_by_modality:
171
+ raise AssertionError(
172
+ f"The processed dummy data has a total of "
173
+ f"{total_placeholders_by_modality} placeholder tokens, which "
174
+ f"is not the expected {expected_placeholders_by_modality} "
175
+ "tokens.")
176
+
177
+ total_len = len(prompt_token_ids)
178
+
179
+ # V0 does not support chunked prefill.
180
+ if total_len > seq_len and not envs.VLLM_USE_V1:
181
+ logger.warning(
182
+ "The context length (%d) of the model is too short "
183
+ "to hold the multi-modal embeddings in the worst case "
184
+ "(%d tokens in total, out of which %s are reserved for "
185
+ "multi-modal embeddings). This may cause certain multi-modal "
186
+ "inputs to fail during inference, even when the input text is "
187
+ "short. To avoid this, you should increase `max_model_len`, "
188
+ "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
189
+ total_len, total_placeholders_by_modality)
190
+
191
+ return DummyData(
192
+ seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
193
+ multi_modal_data=None,
194
+ multi_modal_placeholders=None,
195
+ )
196
+
197
+ prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
198
+
199
+ return DummyData(
200
+ seq_data=SequenceData.from_seqs(prompt_token_ids),
201
+ multi_modal_data=mm_inputs["mm_kwargs"],
202
+ multi_modal_placeholders=placeholders_by_modality,
203
+ )
vllm/multimodal/registry.py
+61
-12
@@ -1,7 +1,8 @@
1
import functools
2
from collections import UserDict
3
-from typing import (TYPE_CHECKING, Any, Dict, Mapping, Optional, Protocol,
4
- Sequence, Type, TypeVar)
3
+from dataclasses import dataclass
4
+from typing import (TYPE_CHECKING, Any, Dict, Generic, Mapping, Optional,
5
+ Protocol, Sequence, Type, TypeVar)
6
7
import torch.nn as nn
8
@@ -14,7 +15,9 @@ from .audio import AudioPlugin
15
from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
16
from .image import ImagePlugin
17
from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
17
-from .processing import BaseMultiModalProcessor, ProcessingCache
18
+from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
19
+ ProcessingCache)
20
+from .profiling import BaseDummyInputsBuilder
21
from .utils import cached_get_tokenizer
22
from .video import VideoPlugin
23
@@ -27,20 +30,59 @@ logger = init_logger(__name__)
30
MM_CACHE_SIZE = 256
31
32
N = TypeVar("N", bound=Type[nn.Module])
33
+_I = TypeVar("_I", bound=BaseProcessingInfo)
34
+_I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
35
36
32
-class MultiModalProcessorFactory(Protocol):
37
+class ProcessingInfoFactory(Protocol[_I_co]):
38
"""Constructs a :class:`MultiModalProcessor` instance from the context."""
39
40
def __call__(
41
self,
42
ctx: InputProcessingContext,
43
+ ) -> _I_co:
44
+ ...
45
+
46
+
47
+class DummyInputsBuilderFactory(Protocol[_I]):
48
+ """
49
+ Constructs a :class:`BaseDummyInputsBuilder` instance from the context.
50
+ """
51
+
52
+ def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]:
53
+ ...
54
+
55
+
56
+class MultiModalProcessorFactory(Protocol[_I]):
57
+ """Constructs a :class:`MultiModalProcessor` instance from the context."""
58
+
59
+ def __call__(
60
+ self,
61
+ info: _I,
62
+ dummy_inputs: BaseDummyInputsBuilder[_I],
63
*,
64
cache: Optional[ProcessingCache] = None,
40
- ) -> BaseMultiModalProcessor:
65
+ ) -> BaseMultiModalProcessor[_I]:
66
...
67
68
69
+@dataclass(frozen=True)
70
+class _ProcessorFactories(Generic[_I]):
71
+ info: ProcessingInfoFactory[_I]
72
+ processor: MultiModalProcessorFactory[_I]
73
+ dummy_inputs: DummyInputsBuilderFactory[_I]
74
+
75
+ def build_processor(
76
+ self,
77
+ ctx: InputProcessingContext,
78
+ *,
79
+ cache: Optional[ProcessingCache] = None,
80
+ ):
81
+ info = self.info(ctx)
82
+ dummy_inputs_builder = self.dummy_inputs(info)
83
+ return self.processor(info, dummy_inputs_builder, cache=cache)
84
+
85
+
86
class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
87
"""
88
Wraps `_limits_by_model` for a more informative error message
@@ -71,7 +113,7 @@ class MultiModalRegistry:
113
self._plugins = {p.get_data_key(): p for p in plugins}
114
115
self._processor_factories = ClassRegistry[nn.Module,
74
- MultiModalProcessorFactory]()
116
+ _ProcessorFactories]()
117
118
# This is used for non-multimodal models
119
self._disabled_limits_per_plugin = {k: 0 for k in self._plugins}
@@ -224,7 +266,7 @@ class MultiModalRegistry:
266
tokenizer = cached_get_tokenizer(model_config.tokenizer)
267
processor = self.create_processor(model_config, tokenizer)
268
seq_len = model_config.max_model_len
227
- return processor.profiling_info.get_mm_max_tokens_per_item(seq_len)
269
+ return processor.info.get_mm_max_tokens_per_item(seq_len)
270
271
return {
272
key: plugin.get_max_multimodal_tokens(model_config)
@@ -315,7 +357,10 @@ class MultiModalRegistry:
357
358
def register_processor(
359
self,
318
- factory: MultiModalProcessorFactory,
360
+ processor: MultiModalProcessorFactory[_I],
361
+ *,
362
+ info: ProcessingInfoFactory[_I],
363
+ dummy_inputs: DummyInputsBuilderFactory[_I],
364
):
365
"""
366
Register a multi-modal processor to a model class. The processor
@@ -336,7 +381,11 @@ class MultiModalRegistry:
381
"registered to %s. It is overwritten by the new one.",
382
model_cls, self)
383
339
- self._processor_factories[model_cls] = factory
384
+ self._processor_factories[model_cls] = _ProcessorFactories(
385
+ info=info,
386
+ dummy_inputs=dummy_inputs,
387
+ processor=processor,
388
+ )
389
390
return model_cls
391
@@ -359,15 +408,15 @@ class MultiModalRegistry:
408
self,
409
model_config: "ModelConfig",
410
tokenizer: AnyTokenizer,
362
- ) -> BaseMultiModalProcessor:
411
+ ) -> BaseMultiModalProcessor[BaseProcessingInfo]:
412
"""
413
Create a multi-modal processor for a specific model and tokenizer.
414
"""
415
model_cls = self._get_model_cls(model_config)
367
- processor_factory = self._processor_factories[model_cls]
416
+ factories = self._processor_factories[model_cls]
417
418
ctx = InputProcessingContext(model_config, tokenizer)
419
cache = (None if model_config.disable_mm_preprocessor_cache else
420
self._processing_cache)
421
373
- return processor_factory(ctx, cache=cache)
422
+ return factories.build_processor(ctx, cache=cache)