A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Support caching in merged multi-modal processor (#11396)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 28, 2024 at 01:22 UTC
101418096ffe3c83b6d541e1303b10e9d5e03861
20 files changed
+1455
-448
docs/source/conf.py
+2
-1
@@ -191,6 +191,7 @@ def linkcode_resolve(domain, info):
191
192
# Mock out external dependencies here, otherwise the autodoc pages may be blank.
193
autodoc_mock_imports = [
194
+ "blake3",
195
"compressed_tensors",
196
"cpuinfo",
197
"cv2",
@@ -207,7 +208,7 @@ autodoc_mock_imports = [
208
"tensorizer",
209
"pynvml",
210
"outlines",
210
- "xgrammar,"
211
+ "xgrammar",
212
"librosa",
213
"soundfile",
214
"gguf",
docs/source/design/multimodal/multimodal_index.md
+12
-12
@@ -45,39 +45,39 @@ adding_multimodal_plugin
45
### Base Classes
46
47
```{eval-rst}
48
-.. autodata:: vllm.multimodal.NestedTensors
48
+.. automodule:: vllm.multimodal.base
49
+ :members:
50
+ :show-inheritance:
51
```
52
51
-```{eval-rst}
52
-.. autodata:: vllm.multimodal.BatchedTensorInputs
53
-```
53
+### Input Classes
54
55
```{eval-rst}
56
-.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
56
+.. automodule:: vllm.multimodal.inputs
57
:members:
58
:show-inheritance:
59
```
60
61
-```{eval-rst}
62
-.. autodata:: vllm.multimodal.MultiModalDataDict
63
-```
61
+### Audio Classes
62
63
```{eval-rst}
66
-.. autoclass:: vllm.multimodal.MultiModalKwargs
64
+.. automodule:: vllm.multimodal.audio
65
:members:
66
:show-inheritance:
67
```
68
69
+### Image Classes
70
+
71
```{eval-rst}
72
-.. autoclass:: vllm.multimodal.MultiModalPlugin
72
+.. automodule:: vllm.multimodal.image
73
:members:
74
:show-inheritance:
75
```
76
77
-### Image Classes
77
+### Video Classes
78
79
```{eval-rst}
80
-.. automodule:: vllm.multimodal.image
80
+.. automodule:: vllm.multimodal.video
81
:members:
82
:show-inheritance:
83
```
docs/source/models/supported_models.md
+1
-2
@@ -755,8 +755,7 @@ vLLM currently only supports adding LoRA to the language backbone of multimodal
755
```
756
757
```{note}
758
-To use {code}`TIGER-Lab/Mantis-8B-siglip-llama3`, you have to install their GitHub repo ({code}`pip install git+https://github.com/TIGER-AI-Lab/Mantis.git`)
759
-and pass {code}`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
758
+To use {code}`TIGER-Lab/Mantis-8B-siglip-llama3`, you have pass {code}`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
759
```
760
761
```{note}
tests/entrypoints/openai/test_vision_embedding.py
+2
-2
@@ -91,5 +91,5 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
91
assert len(embeddings.data) == 1
92
assert len(embeddings.data[0].embedding) == 3072
93
assert embeddings.usage.completion_tokens == 0
94
- assert embeddings.usage.prompt_tokens == 765
95
- assert embeddings.usage.total_tokens == 765
94
+ assert embeddings.usage.prompt_tokens == 764
95
+ assert embeddings.usage.total_tokens == 764
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+1
-1
@@ -30,7 +30,7 @@ def get_max_qwen2_vl_image_tokens():
30
31
32
@pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
33
- ({}, 1225),
33
+ ({}, 16384),
34
({
35
MIN_PIXELS: 64**2,
36
MAX_PIXELS: 512**2
tests/models/decoder_only/vision_language/test_models.py
+3
-1
@@ -201,6 +201,7 @@ VLM_TEST_SETTINGS = {
201
vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
202
num_logprobs=10,
203
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
204
+ marks=[large_gpu_mark(min_gb=48)],
205
),
206
"glm4": VLMTestInfo(
207
models=["THUDM/glm-4v-9b"],
@@ -212,7 +213,7 @@ VLM_TEST_SETTINGS = {
213
dtype="bfloat16",
214
get_stop_token_ids=lambda tok: [151329, 151336, 151338],
215
patch_hf_runner=model_utils.glm_patch_hf_runner,
215
- marks=[large_gpu_mark(min_gb=48)],
216
+ marks=[large_gpu_mark(min_gb=32)],
217
),
218
"h2ovl": VLMTestInfo(
219
models = [
@@ -261,6 +262,7 @@ VLM_TEST_SETTINGS = {
262
dtype="bfloat16",
263
use_tokenizer_eos=True,
264
patch_hf_runner=model_utils.internvl_patch_hf_runner,
265
+ marks=[large_gpu_mark(min_gb=32)],
266
),
267
"llava_next": VLMTestInfo(
268
models=["llava-hf/llava-v1.6-mistral-7b-hf"],
tests/multimodal/test_processing.py
+203
-6
@@ -1,12 +1,20 @@
1
+from functools import partial
2
from typing import cast
3
4
+import numpy as np
5
import pytest
4
-
5
-from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
6
- find_text_matches, find_token_matches,
7
- iter_placeholders, iter_token_matches,
6
+from PIL import Image
7
+
8
+from vllm.config import ModelConfig
9
+from vllm.inputs import InputProcessingContext
10
+from vllm.multimodal import MULTIMODAL_REGISTRY
11
+from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
12
+ _PlaceholderInfo, find_text_matches,
13
+ find_token_matches, iter_placeholders,
14
+ iter_token_matches,
15
replace_text_matches,
16
replace_token_matches)
17
+from vllm.multimodal.utils import cached_get_tokenizer
18
from vllm.transformers_utils.tokenizer import AnyTokenizer
19
from vllm.utils import full_groupby
20
@@ -457,6 +465,7 @@ def test_find_replace_tokens(
465
),
466
]
467
)
468
+# yapf: enable
469
def test_iter_placeholders(
470
repl_by_key,
471
prompt,
@@ -475,11 +484,199 @@ def test_iter_placeholders(
484
prompt_repls,
485
prompt,
486
# Effectively match all occurrences in the prompt
478
- {key: 3 for key in repl_by_key},
479
- ))
487
+ {key: 3
488
+ for key in repl_by_key},
489
+ ))
490
491
# Only displayed on error
492
print("result:", result)
493
494
# Manually constructed results
495
assert result == expected
496
+
497
+
498
+def _rand_img(rng: np.random.RandomState, min_wh: int, max_wh: int):
499
+ w, h = rng.randint(min_wh, max_wh, size=(2, ))
500
+ arr = rng.randint(0, 255, size=(w, h, 3), dtype=np.uint8)
501
+ return Image.fromarray(arr)
502
+
503
+
504
+def _rand_video(
505
+ rng: np.random.RandomState,
506
+ min_frames: int,
507
+ max_frames: int,
508
+ min_wh: int,
509
+ max_wh: int,
510
+):
511
+ # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
512
+ num_frames = rng.randint(min_frames, max_frames)
513
+ num_frames = (num_frames // 2) * 2
514
+
515
+ w, h = rng.randint(min_wh, max_wh, size=(2, ))
516
+ return rng.randint(0, 255, size=(num_frames, w, h, 3), dtype=np.uint8)
517
+
518
+
519
+def _rand_audio(
520
+ rng: np.random.RandomState,
521
+ min_len: int,
522
+ max_len: int,
523
+ sr: int,
524
+):
525
+ audio_len = rng.randint(min_len, max_len)
526
+ return rng.rand(audio_len), sr
527
+
528
+
529
+def _test_processing_cache_correctness(
530
+ model_id: str,
531
+ modalities: set[str],
532
+ hit_rate: float,
533
+ num_batches: int,
534
+ simplify_rate: float,
535
+):
536
+ if model_id == "TIGER-Lab/Mantis-8B-siglip-llama3":
537
+ hf_overrides = {"architectures": ["MantisForConditionalGeneration"]}
538
+ else:
539
+ hf_overrides = {}
540
+
541
+ model_config = ModelConfig(
542
+ model_id,
543
+ task="auto",
544
+ tokenizer=model_id,
545
+ tokenizer_mode="auto",
546
+ trust_remote_code=True,
547
+ seed=0,
548
+ dtype="float16",
549
+ revision=None,
550
+ hf_overrides=hf_overrides,
551
+ )
552
+ model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
553
+
554
+ processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
555
+ ctx = InputProcessingContext(
556
+ model_config,
557
+ tokenizer=cached_get_tokenizer(model_config.tokenizer),
558
+ )
559
+ # Ensure that it can fit all of the data
560
+ cache = ProcessingCache(capacity=1 << 30)
561
+
562
+ baseline_processor = processor_factory(ctx, cache=None)
563
+ cached_processor = processor_factory(ctx, cache=cache)
564
+
565
+ rng = np.random.RandomState(0)
566
+
567
+ input_to_hit = {
568
+ "image": Image.new("RGB", size=(128, 128)),
569
+ "video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
570
+ "audio": (np.zeros((512, )), 16000),
571
+ }
572
+ input_factory = {
573
+ "image":
574
+ partial(_rand_img, rng, min_wh=128, max_wh=256),
575
+ "video":
576
+ partial(_rand_video,
577
+ rng,
578
+ min_frames=2,
579
+ max_frames=8,
580
+ min_wh=128,
581
+ max_wh=256),
582
+ "audio":
583
+ partial(_rand_audio, rng, min_len=256, max_len=512, sr=16000),
584
+ }
585
+ input_max_count = {
586
+ "image": 3,
587
+ "video": 3,
588
+ "audio": 3,
589
+ }
590
+
591
+ for batch_idx in range(num_batches):
592
+ mm_data = {
593
+ k:
594
+ [(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
595
+ for _ in range(rng.randint(input_max_count[k]))]
596
+ for k in modalities
597
+ }
598
+
599
+ mm_counts = {k: len(vs) for k, vs in mm_data.items()}
600
+ prompt = baseline_processor._get_dummy_mm_inputs(mm_counts).prompt_text
601
+
602
+ # Drop unnecessary keys and test single -> multi conversion
603
+ if rng.rand() < simplify_rate:
604
+ for k in list(mm_data.keys()):
605
+ if not mm_data[k]:
606
+ del mm_data[k]
607
+ elif len(mm_data[k]) == 1:
608
+ mm_data[k] = mm_data[k][0]
609
+
610
+ baseline_result = baseline_processor.apply(
611
+ prompt,
612
+ mm_data=mm_data,
613
+ hf_processor_mm_kwargs={},
614
+ )
615
+ cached_result = cached_processor.apply(
616
+ prompt,
617
+ mm_data=mm_data,
618
+ hf_processor_mm_kwargs={},
619
+ )
620
+
621
+ assert baseline_result == cached_result, (
622
+ f"Failed ({batch_idx=}, {mm_data=})")
623
+
624
+
625
+# yapf: disable
626
+@pytest.mark.parametrize(("model_id", "modalities"), [
627
+ ("llava-hf/llava-1.5-7b-hf", {"image"}),
628
+ ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image"}),
629
+ ("mistral-community/pixtral-12b", {"image"}),
630
+ ("Qwen/Qwen2-VL-2B-Instruct", {"image", "video"}),
631
+ ("Qwen/Qwen2-Audio-7B-Instruct", {"audio"}),
632
+ ("fixie-ai/ultravox-v0_3", {"audio"}),
633
+])
634
+@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
635
+@pytest.mark.parametrize("num_batches", [32])
636
+@pytest.mark.parametrize("simplify_rate", [1.0])
637
+# yapf: enable
638
+def test_processing_cache_correctness(
639
+ model_id: str,
640
+ modalities: set[str],
641
+ hit_rate: float,
642
+ num_batches: int,
643
+ simplify_rate: float,
644
+):
645
+ _test_processing_cache_correctness(
646
+ model_id,
647
+ modalities,
648
+ hit_rate=hit_rate,
649
+ num_batches=num_batches,
650
+ simplify_rate=simplify_rate,
651
+ )
652
+
653
+
654
+# yapf: disable
655
+@pytest.mark.parametrize(("model_id", "modalities"), [
656
+ ("microsoft/Phi-3-vision-128k-instruct", {"image"}),
657
+])
658
+@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
659
+@pytest.mark.parametrize("num_batches", [32])
660
+@pytest.mark.parametrize("simplify_rate", [1.0])
661
+# yapf: enable
662
+def test_processing_cache_correctness_phi3v(
663
+ model_id: str,
664
+ modalities: set[str],
665
+ hit_rate: float,
666
+ num_batches: int,
667
+ simplify_rate: float,
668
+):
669
+ # HACK - this is an attempted workaround for the following bug
670
+ # https://github.com/huggingface/transformers/issues/34307
671
+ from transformers import AutoImageProcessor # noqa: F401
672
+ from transformers import AutoProcessor # noqa: F401
673
+
674
+ AutoImageProcessor.from_pretrained(model_id, trust_remote_code=True)
675
+
676
+ _test_processing_cache_correctness(
677
+ model_id,
678
+ modalities,
679
+ hit_rate=hit_rate,
680
+ num_batches=num_batches,
681
+ simplify_rate=simplify_rate,
682
+ )
vllm/inputs/registry.py
+11
-11
@@ -99,6 +99,9 @@ class InputContext:
99
100
merged_kwargs = {**base_kwargs, **kwargs}
101
102
+ if isinstance(typ, type):
103
+ merged_kwargs["processor_cls"] = typ
104
+
105
hf_processor = cached_get_processor(
106
self.model_config.model,
107
trust_remote_code=self.model_config.trust_remote_code,
@@ -132,10 +135,13 @@ class InputProcessingContext(InputContext):
135
def call_hf_processor(
136
self,
137
hf_processor: ProcessorMixin,
135
- prompt: str,
136
- processor_data: Mapping[str, object],
137
- inference_kwargs: Mapping[str, object],
138
+ data: Mapping[str, object],
139
+ kwargs: Mapping[str, object] = {},
140
) -> BatchFeature:
141
+ """
142
+ Call :code:`hf_processor` on the prompt :code:`data`
143
+ (text, image, audio...) with configurable options :code:`kwargs`.
144
+ """
145
assert callable(hf_processor)
146
147
base_kwargs = self.model_config.mm_processor_kwargs
@@ -144,21 +150,15 @@ class InputProcessingContext(InputContext):
150
151
merged_kwargs = resolve_mm_processor_kwargs(
152
base_kwargs,
147
- inference_kwargs,
153
+ kwargs,
154
hf_processor,
155
requires_kw_only=False,
156
allow_var_kwargs=True,
157
)
158
159
try:
154
- return hf_processor(
155
- text=prompt,
156
- **processor_data,
157
- **merged_kwargs,
158
- return_tensors="pt",
159
- )
160
+ return hf_processor(**data, **merged_kwargs, return_tensors="pt")
161
except Exception as exc:
161
- data = dict(text=prompt, **processor_data)
162
msg = (f"Failed to apply {type(hf_processor).__name__} "
163
f"on data={data} with kwargs={merged_kwargs}")
164
vllm/model_executor/models/llava.py
+110
-68
@@ -1,5 +1,4 @@
1
from functools import cached_property
2
-from types import MethodType
2
from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
3
Tuple, TypedDict, Union)
4
@@ -7,7 +6,7 @@ import torch
6
import torch.nn as nn
7
from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
8
PixtralVisionConfig, PretrainedConfig,
10
- ProcessorMixin, SiglipVisionConfig)
9
+ SiglipVisionConfig)
10
from transformers.models.llava import LlavaProcessor
11
from transformers.models.pixtral import PixtralProcessor
12
@@ -21,10 +20,12 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.multimodal import MULTIMODAL_REGISTRY
24
-from vllm.multimodal.inputs import NestedTensors
23
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
24
+ MultiModalFieldConfig, MultiModalInputsV2,
25
+ MultiModalKwargs, NestedTensors)
26
from vllm.multimodal.processing import (BaseMultiModalProcessor,
26
- MultiModalDataItems, ProcessorInputs,
27
- PromptReplacement)
27
+ ProcessorInputs, PromptReplacement,
28
+ full_groupby_modality)
29
from vllm.sequence import IntermediateTensors
30
31
from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -116,36 +117,54 @@ def get_max_llava_image_tokens(ctx: InputContext):
117
118
class LlavaMultiModalProcessor(BaseMultiModalProcessor):
119
119
- def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
120
- if getattr(hf_processor, "__is_patched__", False):
121
- return # Already patched
122
-
123
- image_processor = hf_processor.image_processor # type: ignore
124
- orig_preprocess = image_processor.preprocess
120
+ def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
121
+ return self.ctx.get_hf_processor((LlavaProcessor, PixtralProcessor))
122
126
- def preprocess(__self, *args, **kwargs):
127
- hf_inputs = orig_preprocess(*args, **kwargs)
128
- hf_inputs["is_pixtral"] = torch.tensor(True)
129
- return hf_inputs
123
+ def _call_hf_processor(
124
+ self,
125
+ prompt: str,
126
+ mm_data: Mapping[str, object],
127
+ mm_kwargs: Mapping[str, object],
128
+ ) -> BatchFeature:
129
+ processed_outputs = super()._call_hf_processor(
130
+ prompt=prompt,
131
+ mm_data=mm_data,
132
+ mm_kwargs=mm_kwargs,
133
+ )
134
131
- image_processor.preprocess = MethodType(preprocess, image_processor)
135
+ # NOTE: pixel_values=None for MLlavaProcessor
136
+ pixel_values = processed_outputs.get("pixel_values")
137
+ if pixel_values is not None:
138
+ images = mm_data["images"]
139
+ assert isinstance(images, list)
140
133
- hf_processor.__is_patched__ = True # type: ignore
141
+ if isinstance(self._get_hf_processor(), PixtralProcessor):
142
+ # Original output: (1, num_images, C, H, W)
143
+ # New output: (num_images, C, H, W)
144
+ assert (isinstance(pixel_values, list)
145
+ and len(pixel_values) == 1
146
+ and isinstance(pixel_values[0], list)
147
+ and len(pixel_values[0]) == len(images))
148
135
- def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
136
- hf_processor = self.ctx.get_hf_processor(
137
- (LlavaProcessor, PixtralProcessor))
149
+ processed_outputs["pixel_values"] = pixel_values[0]
150
139
- if isinstance(hf_processor, PixtralProcessor):
140
- self._patch_pixtral_processor(hf_processor)
151
+ return processed_outputs
152
142
- return hf_processor
153
+ def _get_mm_fields_config(
154
+ self,
155
+ hf_inputs: BatchFeature,
156
+ hf_processor_mm_kwargs: Mapping[str, object],
157
+ ) -> Mapping[str, MultiModalFieldConfig]:
158
+ return dict(
159
+ pixel_values=MultiModalFieldConfig.batched("image"),
160
+ image_embeds=MultiModalFieldConfig.batched("image"),
161
+ )
162
163
def _get_prompt_replacements(
164
self,
165
mm_items: MultiModalDataItems,
147
- hf_inputs: BatchFeature,
148
- mm_processor_kwargs: Mapping[str, object],
166
+ hf_processor_mm_kwargs: Mapping[str, object],
167
+ out_mm_kwargs: MultiModalKwargs,
168
) -> list[PromptReplacement]:
169
hf_config = self.ctx.get_hf_config(LlavaConfig)
170
image_token_id = hf_config.image_token_index
@@ -200,7 +219,7 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
219
) -> ProcessorInputs:
220
hf_config = self.ctx.get_hf_config(LlavaConfig)
221
vision_config = hf_config.vision_config
203
- num_images = mm_counts["image"]
222
+ num_images = mm_counts.get("image", 0)
223
224
if isinstance(vision_config, CLIPVisionConfig):
225
data = dummy_image_for_clip(vision_config, num_images)
@@ -218,7 +237,6 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
237
return ProcessorInputs(
238
prompt_text=image_token * num_images,
239
mm_data=data,
221
- mm_processor_kwargs={},
240
)
241
242
@@ -379,7 +397,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
397
def _parse_and_validate_image_input(
398
self, **kwargs: object) -> Optional[LlavaImageInputs]:
399
pixel_values = kwargs.pop("pixel_values", None)
382
- is_pixtral = kwargs.pop("is_pixtral", torch.tensor([False]))
400
image_embeds = kwargs.pop("image_embeds", None)
401
402
if pixel_values is None and image_embeds is None:
@@ -390,33 +407,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
407
raise ValueError("Incorrect type of pixel values. "
408
f"Got type: {type(pixel_values)}")
409
393
- assert isinstance(is_pixtral, torch.Tensor)
394
- if is_pixtral.any():
395
- images = pixel_values
396
-
397
- def flatten_to_3d_tensors(item):
398
- if isinstance(item, torch.Tensor):
399
- if item.dim() >= 3:
400
- return [t for t in item.view(-1, *item.shape[-3:])]
401
- else:
402
- raise ValueError(
403
- f"Unexpected tensor dimension: {item.dim()}")
404
- elif isinstance(item, list):
405
- return [
406
- t for subitem in item
407
- for t in flatten_to_3d_tensors(subitem)
408
- ]
409
- else:
410
- raise ValueError(f"Unexpected type: {type(item)}")
411
-
412
- # Restructure the batched images into a list of lists of images
413
- images = flatten_to_3d_tensors(pixel_values)
414
-
415
- return LlavaImagePixelInputs(
416
- type="pixel_values",
417
- data=images,
418
- )
419
-
410
return LlavaImagePixelInputs(
411
type="pixel_values",
412
data=self._validate_pixel_values(
@@ -586,19 +576,71 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
576
577
class MantisMultiModalProcessor(LlavaMultiModalProcessor):
578
589
- def _get_hf_processor(self) -> ProcessorMixin:
590
- try:
591
- from mantis.models.mllava import MLlavaProcessor
592
- except ModuleNotFoundError as exc:
593
- raise ModuleNotFoundError(
594
- "You need to `pip install "
595
- "git+https://github.com/TIGER-AI-Lab/Mantis.git` "
596
- "to use this model") from exc
597
-
598
- processor = MLlavaProcessor.from_pretrained(
599
- self.ctx.model_config.tokenizer)
600
- assert isinstance(processor, ProcessorMixin)
601
- return processor
579
+ def _get_hf_processor(self):
580
+ return self.ctx.get_hf_processor(LlavaProcessor)
581
+
582
+ def apply(
583
+ self,
584
+ prompt_text: str,
585
+ mm_data: MultiModalDataDict,
586
+ hf_processor_mm_kwargs: Mapping[str, object],
587
+ ) -> MultiModalInputsV2:
588
+ hf_config = self.ctx.get_hf_config(LlavaConfig)
589
+ image_token_id = hf_config.image_token_index
590
+ max_image_tokens = get_max_llava_image_tokens(self.ctx)
591
+
592
+ result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
593
+
594
+ mm_items = self._get_mm_items(mm_data)
595
+ mm_item_counts = mm_items.get_item_counts()
596
+ mm_kwargs = result["mm_kwargs"]
597
+
598
+ # We reimplement the functionality of MLlavaProcessor from
599
+ # https://github.com/TIGER-AI-Lab/Mantis.git
600
+ def get_replacement_mantis(item_idx: int):
601
+ return "".join([
602
+ f"(image {item_idx+1}: <Image>", # 7 tokens
603
+ "<image>" * max_image_tokens,
604
+ "</Image>)", # 3 tokens
605
+ ])
606
+
607
+ mantis_repls = self._bind_prompt_replacements([
608
+ PromptReplacement(
609
+ modality="image",
610
+ target=[image_token_id] * max_image_tokens,
611
+ replacement=get_replacement_mantis,
612
+ )
613
+ ])
614
+
615
+ prompt_ids, prompt_text, _ = self._apply_prompt_replacements(
616
+ result["prompt_token_ids"],
617
+ mantis_repls,
618
+ mm_item_counts,
619
+ )
620
+
621
+ unbound_orig_repls = self._get_prompt_replacements(
622
+ mm_items,
623
+ hf_processor_mm_kwargs,
624
+ mm_kwargs,
625
+ )
626
+ orig_repls = self._bind_prompt_replacements(unbound_orig_repls)
627
+
628
+ all_placeholders = self._find_placeholders(orig_repls, prompt_ids,
629
+ mm_item_counts)
630
+ assert len(all_placeholders) == mm_item_counts.get("image", 0)
631
+
632
+ mm_placeholders = {
633
+ modality: [item.to_range() for item in items]
634
+ for modality, items in full_groupby_modality(all_placeholders)
635
+ }
636
+
637
+ return MultiModalInputsV2(
638
+ type="multimodal",
639
+ prompt=prompt_text,
640
+ prompt_token_ids=prompt_ids,
641
+ mm_kwargs=mm_kwargs,
642
+ mm_placeholders=mm_placeholders,
643
+ )
644
645
646
# To use this model, please use
vllm/model_executor/models/phi3v.py
+82
-25
@@ -12,9 +12,9 @@
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
+from collections.abc import Iterable, Mapping, Sequence
16
from functools import cached_property
16
-from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
17
- TypedDict, Union)
17
+from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
18
19
import torch
20
import torch.nn as nn
@@ -32,10 +32,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
from vllm.model_executor.models.clip import CLIPVisionModel
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.multimodal import MULTIMODAL_REGISTRY
35
-from vllm.multimodal.inputs import NestedTensors
35
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
36
+ MultiModalFieldConfig, MultiModalInputsV2,
37
+ MultiModalKwargs, NestedTensors,
38
+ PlaceholderRange)
39
from vllm.multimodal.processing import (BaseMultiModalProcessor,
37
- MultiModalDataItems, ProcessorInputs,
38
- PromptReplacement)
40
+ ProcessorInputs, PromptReplacement,
41
+ _BoundPromptReplacement,
42
+ _PlaceholderInfo)
43
from vllm.sequence import IntermediateTensors
44
from vllm.utils import is_list_of
45
@@ -306,11 +310,11 @@ def get_max_phi3v_image_tokens(
310
*,
311
num_crops: Optional[int] = None,
312
) -> int:
309
- mm_processor_kwargs = {}
313
+ hf_processor_mm_kwargs = {}
314
if num_crops:
311
- mm_processor_kwargs["num_crops"] = num_crops
315
+ hf_processor_mm_kwargs["num_crops"] = num_crops
316
313
- processor = ctx.get_hf_processor(**mm_processor_kwargs)
317
+ processor = ctx.get_hf_processor(**hf_processor_mm_kwargs)
318
319
return processor.calc_num_image_tokens_from_image_size(
320
width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
@@ -331,39 +335,50 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
335
336
def _call_hf_processor(
337
self,
334
- hf_processor: ProcessorMixin,
338
prompt: str,
336
- processor_data: Mapping[str, object],
337
- mm_processor_kwargs: Mapping[str, object],
339
+ mm_data: Mapping[str, object],
340
+ mm_kwargs: Mapping[str, object],
341
) -> BatchFeature:
342
processed_outputs = super()._call_hf_processor(
340
- hf_processor,
343
prompt=prompt,
342
- processor_data=processor_data,
343
- mm_processor_kwargs=mm_processor_kwargs,
344
+ mm_data=mm_data,
345
+ mm_kwargs=mm_kwargs,
346
)
347
348
+ input_ids = processed_outputs["input_ids"]
349
+ assert isinstance(input_ids, torch.Tensor)
350
+
351
# Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
352
# which will cause OverflowError when decoding the prompt_ids.
353
# Therefore, we need to do an early replacement here
349
- token_ids = processed_outputs['input_ids']
350
- token_ids[token_ids < 0] = _IMAGE_TOKEN_ID
351
- processed_outputs['input_ids'] = token_ids
354
+ input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
355
356
return processed_outputs
357
358
+ def _get_mm_fields_config(
359
+ self,
360
+ hf_inputs: BatchFeature,
361
+ hf_processor_mm_kwargs: Mapping[str, object],
362
+ ) -> Mapping[str, MultiModalFieldConfig]:
363
+ return dict(
364
+ pixel_values=MultiModalFieldConfig.batched("image"),
365
+ image_sizes=MultiModalFieldConfig.batched("image"),
366
+ image_embeds=MultiModalFieldConfig.batched("image"),
367
+ )
368
+
369
def _get_prompt_replacements(
370
self,
371
mm_items: MultiModalDataItems,
358
- hf_inputs: BatchFeature,
359
- mm_processor_kwargs: Mapping[str, object],
372
+ hf_processor_mm_kwargs: Mapping[str, object],
373
+ out_mm_kwargs: MultiModalKwargs,
374
) -> list[PromptReplacement]:
375
hf_processor = self._get_hf_processor()
376
image_tokens: list[str] = hf_processor.img_tokens # type: ignore
377
image_processor = hf_processor.image_processor # type: ignore
378
365
- mm_config = self.ctx.get_mm_config()
366
- max_images = mm_config.limit_per_prompt.get("image", 1)
379
+ tokenizer = self._get_tokenizer()
380
+ bos_token_id = tokenizer.bos_token_id
381
+ assert isinstance(bos_token_id, int)
382
383
def get_replacement_phi3v(item_idx: int):
384
image_size = mm_items.get_image_size(item_idx)
@@ -372,21 +387,44 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
387
height=image_size.height,
388
)
389
375
- return [_IMAGE_TOKEN_ID] * num_tokens
390
+ return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
391
392
return [
393
PromptReplacement(
394
modality="image",
395
target=image_token,
396
replacement=get_replacement_phi3v,
382
- ) for image_token in image_tokens[:max_images]
397
+ ) for image_token in image_tokens[:len(mm_items.images)]
398
]
399
400
+ def _apply_prompt_replacements(
401
+ self,
402
+ token_ids: list[int],
403
+ prompt_repls: Sequence[_BoundPromptReplacement],
404
+ mm_item_counts: Mapping[str, int],
405
+ ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
406
+ token_ids, text, placeholders = super()._apply_prompt_replacements(
407
+ token_ids=token_ids,
408
+ prompt_repls=prompt_repls,
409
+ mm_item_counts=mm_item_counts,
410
+ )
411
+
412
+ # Keep the behavior in line with HF processor
413
+ if text.startswith("<s> <|image|>"):
414
+ text = text.replace("<s> <|image|>", "<s><|image|>", 1)
415
+ token_ids = [token_ids[0], *token_ids[2:]]
416
+ placeholders = [
417
+ _PlaceholderInfo(p.modality, p.start_idx - 1, p.replacement)
418
+ for p in placeholders
419
+ ]
420
+
421
+ return token_ids, text, placeholders
422
+
423
def _get_dummy_mm_inputs(
424
self,
425
mm_counts: Mapping[str, int],
426
) -> ProcessorInputs:
389
- num_images = mm_counts["image"]
427
+ num_images = mm_counts.get("image", 0)
428
429
data = dummy_image_for_clip(
430
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
@@ -401,9 +439,28 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
439
return ProcessorInputs(
440
prompt_text="".join(image_tokens[:num_images]),
441
mm_data=data,
404
- mm_processor_kwargs={},
442
)
443
444
+ def apply(
445
+ self,
446
+ prompt_text: str,
447
+ mm_data: MultiModalDataDict,
448
+ hf_processor_mm_kwargs: Mapping[str, object],
449
+ ) -> MultiModalInputsV2:
450
+ result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
451
+
452
+ # Only <|image|> tokens should be considered as placeholders,
453
+ # so we ignore the trailing bos_token_id
454
+ result["mm_placeholders"] = {
455
+ modality: [
456
+ PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
457
+ for p in ps
458
+ ]
459
+ for modality, ps in result["mm_placeholders"].items()
460
+ }
461
+
462
+ return result
463
+
464
465
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
466
@MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
vllm/model_executor/models/qwen.py
+2
-2
@@ -225,7 +225,7 @@ class VisualAttentionBlock(nn.Module):
225
d_model: int,
226
n_head: int,
227
mlp_ratio: float = 4.0,
228
- norm_layer: Callable = nn.LayerNorm,
228
+ norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
229
quant_config: Optional[QuantizationConfig] = None,
230
):
231
super().__init__()
@@ -266,7 +266,7 @@ class TransformerBlock(nn.Module):
266
layers: int,
267
heads: int,
268
mlp_ratio: float = 4.0,
269
- norm_layer: Callable = nn.LayerNorm,
269
+ norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
270
quant_config: Optional[QuantizationConfig] = None,
271
):
272
super().__init__()
vllm/model_executor/models/qwen2_audio.py
+40
-25
@@ -26,7 +26,7 @@ from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
26
import numpy as np
27
import torch
28
import torch.nn as nn
29
-from transformers import BatchFeature, ProcessorMixin
29
+from transformers import BatchFeature
30
from transformers.models.qwen2_audio import (Qwen2AudioConfig,
31
Qwen2AudioEncoder,
32
Qwen2AudioProcessor)
@@ -38,10 +38,10 @@ from vllm.inputs import InputContext
38
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39
from vllm.model_executor.sampling_metadata import SamplingMetadata
40
from vllm.multimodal import MULTIMODAL_REGISTRY
41
-from vllm.multimodal.inputs import NestedTensors
41
+from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
42
+ MultiModalKwargs, NestedTensors)
43
from vllm.multimodal.processing import (BaseMultiModalProcessor,
43
- MultiModalDataItems, ProcessorInputs,
44
- PromptReplacement)
44
+ ProcessorInputs, PromptReplacement)
45
from vllm.sequence import IntermediateTensors
46
47
from .interfaces import SupportsMultiModal, SupportsPP
@@ -73,7 +73,7 @@ class Qwen2AudioMultiModalProjector(nn.Module):
73
74
75
# From Qwen2AudioEncoder._get_feat_extract_output_lengths
76
-def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
76
+def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
77
feat_lengths = (input_lengths - 1) // 2 + 1
78
output_lengths = (feat_lengths - 2) // 2 + 1
79
return feat_lengths, output_lengths
@@ -88,13 +88,18 @@ def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
88
89
class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
90
91
- def _get_hf_processor(self) -> Qwen2AudioProcessor:
91
+ def _get_hf_processor(
92
+ self,
93
+ *,
94
+ # Ignored in initialization
95
+ sampling_rate: Optional[int] = None,
96
+ ) -> Qwen2AudioProcessor:
97
return self.ctx.get_hf_processor(Qwen2AudioProcessor)
98
99
def _get_feature_extractor(self) -> WhisperFeatureExtractor:
100
return self._get_hf_processor().feature_extractor # type: ignore
101
97
- def _get_processor_data(
102
+ def _get_hf_mm_data(
103
self,
104
mm_items: MultiModalDataItems,
105
) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -102,50 +107,61 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
107
feature_extractor = self._get_feature_extractor()
108
mm_items.resample_audios(feature_extractor.sampling_rate)
109
105
- return super()._get_processor_data(mm_items)
110
+ return super()._get_hf_mm_data(mm_items)
111
112
def _call_hf_processor(
113
self,
109
- hf_processor: ProcessorMixin,
114
prompt: str,
111
- processor_data: Mapping[str, object],
112
- mm_processor_kwargs: Mapping[str, object],
115
+ mm_data: Mapping[str, object],
116
+ mm_kwargs: Mapping[str, object],
117
) -> BatchFeature:
114
- processor_data = dict(processor_data)
115
- audios = processor_data.pop("audios", [])
118
+ mm_data = dict(mm_data)
119
+ audios = mm_data.pop("audios", [])
120
121
if audios:
118
- processor_data["audios"] = audios
122
+ mm_data["audios"] = audios
123
124
feature_extractor = self._get_feature_extractor()
121
- mm_processor_kwargs = dict(
122
- **mm_processor_kwargs,
125
+ mm_kwargs = dict(
126
+ **mm_kwargs,
127
sampling_rate=feature_extractor.sampling_rate,
128
)
129
else:
130
# NOTE: WhisperFeatureExtractor cannot handle empty list of audios
131
pass
132
129
- return super()._call_hf_processor(
130
- hf_processor,
133
+ processed_outputs = super()._call_hf_processor(
134
prompt=prompt,
132
- processor_data=processor_data,
133
- mm_processor_kwargs=mm_processor_kwargs,
135
+ mm_data=mm_data,
136
+ mm_kwargs=mm_kwargs,
137
+ )
138
+
139
+ return processed_outputs
140
+
141
+ def _get_mm_fields_config(
142
+ self,
143
+ hf_inputs: BatchFeature,
144
+ hf_processor_mm_kwargs: Mapping[str, object],
145
+ ) -> Mapping[str, MultiModalFieldConfig]:
146
+ return dict(
147
+ input_features=MultiModalFieldConfig.batched("audio"),
148
+ feature_attention_mask=MultiModalFieldConfig.batched("audio"),
149
)
150
151
def _get_prompt_replacements(
152
self,
153
mm_items: MultiModalDataItems,
139
- hf_inputs: BatchFeature,
140
- mm_processor_kwargs: Mapping[str, object],
154
+ hf_processor_mm_kwargs: Mapping[str, object],
155
+ out_mm_kwargs: MultiModalKwargs,
156
) -> list[PromptReplacement]:
157
hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
158
placeholder = hf_config.audio_token_index
159
145
- feature_attention_mask = hf_inputs.get("feature_attention_mask")
160
+ feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
161
if feature_attention_mask is None:
162
audio_output_lengths = []
163
else:
164
+ assert isinstance(feature_attention_mask, torch.Tensor)
165
_, audio_output_lengths = _get_feat_extract_output_lengths(
166
feature_attention_mask.sum(-1))
167
@@ -168,14 +184,13 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
184
sampling_rate = feature_extractor.sampling_rate
185
audio_len = feature_extractor.chunk_length * sampling_rate
186
171
- audio_count = mm_counts["audio"]
187
+ audio_count = mm_counts.get("audio", 0)
188
audio = np.zeros(audio_len)
189
data = {"audio": [audio] * audio_count}
190
191
return ProcessorInputs(
192
prompt_text="<|AUDIO|>" * audio_count,
193
mm_data=data,
178
- mm_processor_kwargs={},
194
)
195
196
vllm/model_executor/models/qwen2_vl.py
+78
-37
@@ -22,9 +22,10 @@
22
# limitations under the License.
23
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24
from functools import cached_property, partial
25
-from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
26
- Tuple, Type, TypedDict, Union)
25
+from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
26
+ Set, Tuple, Type, TypedDict, Union)
27
28
+import numpy as np
29
import torch
30
import torch.nn as nn
31
import torch.nn.functional as F
@@ -54,10 +55,11 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
55
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
56
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
57
from vllm.multimodal import MULTIMODAL_REGISTRY
57
-from vllm.multimodal.inputs import MultiModalDataDict, NestedTensors
58
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
59
+ MultiModalFieldConfig, MultiModalKwargs,
60
+ NestedTensors)
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
59
- MultiModalDataItems, ProcessorInputs,
60
- PromptReplacement)
62
+ ProcessorInputs, PromptReplacement)
63
from vllm.platforms import _Backend
64
from vllm.sequence import IntermediateTensors
65
from vllm.transformers_utils.config import uses_mrope
@@ -229,9 +231,9 @@ class Qwen2VisionAttention(nn.Module):
231
232
def __init__(
233
self,
232
- embed_dim: Optional[int] = None,
233
- num_heads: Optional[int] = None,
234
- projection_size: Optional[int] = None,
234
+ embed_dim: int,
235
+ num_heads: int,
236
+ projection_size: int,
237
quant_config: Optional[QuantizationConfig] = None,
238
prefix: str = "",
239
) -> None:
@@ -264,7 +266,7 @@ class Qwen2VisionAttention(nn.Module):
266
self,
267
x: torch.Tensor,
268
cu_seqlens: torch.Tensor,
267
- rotary_pos_emb: torch.Tensor = None,
269
+ rotary_pos_emb: torch.Tensor,
270
) -> torch.Tensor:
271
# [s, b, c] --> [s, b, head * 3 * head_dim]
272
x, _ = self.qkv(x)
@@ -347,7 +349,7 @@ class Qwen2VisionBlock(nn.Module):
349
num_heads: int,
350
mlp_ratio: float,
351
act_layer: Type[nn.Module] = QuickGELU,
350
- norm_layer: Type[nn.Module] = None,
352
+ norm_layer: Optional[Callable[[int], nn.Module]] = None,
353
quant_config: Optional[QuantizationConfig] = None,
354
prefix: str = "",
355
) -> None:
@@ -384,7 +386,7 @@ class Qwen2VisionPatchEmbed(nn.Module):
386
self,
387
patch_size: int = 14,
388
temporal_patch_size: int = 2,
387
- in_chans: int = 3,
389
+ in_channels: int = 3,
390
embed_dim: int = 1152,
391
) -> None:
392
super().__init__()
@@ -392,8 +394,8 @@ class Qwen2VisionPatchEmbed(nn.Module):
394
self.temporal_patch_size = temporal_patch_size
395
self.embed_dim = embed_dim
396
395
- kernel_size = [temporal_patch_size, patch_size, patch_size]
396
- self.proj = nn.Conv3d(in_chans,
397
+ kernel_size = (temporal_patch_size, patch_size, patch_size)
398
+ self.proj = nn.Conv3d(in_channels,
399
embed_dim,
400
kernel_size=kernel_size,
401
stride=kernel_size,
@@ -413,7 +415,7 @@ class Qwen2VisionPatchMerger(nn.Module):
415
self,
416
d_model: int,
417
context_dim: int,
416
- norm_layer: Type[nn.Module] = None,
418
+ norm_layer: Optional[Callable[[int], nn.Module]] = None,
419
spatial_merge_size: int = 2,
420
quant_config: Optional[QuantizationConfig] = None,
421
prefix: str = "",
@@ -489,15 +491,15 @@ class Qwen2VisionTransformer(nn.Module):
491
) -> None:
492
super().__init__()
493
492
- patch_size: int = vision_config.patch_size
493
- temporal_patch_size: int = vision_config.temporal_patch_size
494
- spatial_merge_size: int = vision_config.spatial_merge_size
495
- in_chans: int = vision_config.in_chans
496
- hidden_size: int = vision_config.hidden_size
497
- embed_dim: int = vision_config.embed_dim
498
- depth: int = vision_config.depth
499
- num_heads: int = vision_config.num_heads
500
- mlp_ratio: float = vision_config.mlp_ratio
494
+ patch_size = vision_config.patch_size
495
+ temporal_patch_size = vision_config.temporal_patch_size
496
+ spatial_merge_size = vision_config.spatial_merge_size
497
+ in_channels = vision_config.in_channels
498
+ hidden_size = vision_config.hidden_size
499
+ embed_dim = vision_config.embed_dim
500
+ depth = vision_config.depth
501
+ num_heads = vision_config.num_heads
502
+ mlp_ratio = vision_config.mlp_ratio
503
504
self.spatial_merge_size = spatial_merge_size
505
self.num_heads = num_heads
@@ -506,7 +508,7 @@ class Qwen2VisionTransformer(nn.Module):
508
self.patch_embed = Qwen2VisionPatchEmbed(
509
patch_size=patch_size,
510
temporal_patch_size=temporal_patch_size,
509
- in_chans=in_chans,
511
+ in_channels=in_channels,
512
embed_dim=embed_dim,
513
)
514
@@ -733,8 +735,12 @@ class Qwen2VLMultiModalDataItems(MultiModalDataItems):
735
if k == "video":
736
# Special case since even a single item can be a list
737
multi_data[k] = ( # type: ignore[index]
736
- v if (isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
737
- or is_list_of(v, list)) else [v]
738
+ v if (
739
+ isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
740
+ or is_list_of(v, list)
741
+ or isinstance(v[0], (np.ndarray, torch.Tensor))
742
+ and v[0].ndim == 4
743
+ ) else [v]
744
)
745
elif k in ("image", "audio"):
746
multi_data[k] = ( # type: ignore[index]
@@ -754,6 +760,12 @@ class Qwen2VLMultiModalDataItems(MultiModalDataItems):
760
for m, items in self.items()
761
}
762
763
+ def has_embedding_inputs(self) -> bool:
764
+ return any(
765
+ isinstance(items, dict) or any(
766
+ isinstance(item, torch.Tensor) for item in items)
767
+ for items in self.values())
768
+
769
770
class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
771
@@ -784,7 +796,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
796
797
return hf_processor
798
787
- def _get_processor_data(
799
+ def _get_hf_mm_data(
800
self,
801
mm_items: MultiModalDataItems,
802
) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -805,7 +817,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
817
and v[0].ndim == 2):
818
# Pass through embedding inputs (multi)
819
passthrough_data[f"{k}_embeds"] = v
808
- else:
820
+ elif len(v) > 0:
821
# Map keys to plural form, e.g.: image -> images
822
processor_data[f"{k}s"] = v
823
else:
@@ -816,8 +828,8 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
828
def _get_prompt_replacements(
829
self,
830
mm_items: MultiModalDataItems,
819
- hf_inputs: BatchFeature,
820
- mm_processor_kwargs: Mapping[str, object],
831
+ hf_processor_mm_kwargs: Mapping[str, object],
832
+ out_mm_kwargs: MultiModalKwargs,
833
) -> list[PromptReplacement]:
834
hf_processor = self._get_hf_processor()
835
image_processor = _get_image_processor(hf_processor)
@@ -831,7 +843,9 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
843
merge_length = image_processor.merge_size**2
844
845
def get_replacement_qwen2vl(item_idx: int, modality: str):
834
- grid_thw = hf_inputs[f"{modality}_grid_thw"][item_idx]
846
+ grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
847
+ assert isinstance(grid_thw, torch.Tensor)
848
+
849
num_tokens = grid_thw.prod() // merge_length
850
return placeholder[modality] * num_tokens
851
@@ -844,11 +858,40 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
858
) for modality in ("image", "video")
859
]
860
861
+ def _get_mm_fields_config(
862
+ self,
863
+ hf_inputs: BatchFeature,
864
+ hf_processor_mm_kwargs: Mapping[str, object],
865
+ ) -> Mapping[str, MultiModalFieldConfig]:
866
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
867
+ image_slice_idxs = [0] + image_grid_thw.prod(-1).cumsum_(0).tolist()
868
+ image_slices = [
869
+ slice(image_slice_idxs[i], image_slice_idxs[i + 1])
870
+ for i in range(len(image_grid_thw))
871
+ ]
872
+
873
+ video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
874
+ video_slice_idxs = [0] + video_grid_thw.prod(-1).cumsum_(0).tolist()
875
+ video_slices = [
876
+ slice(video_slice_idxs[i], video_slice_idxs[i + 1])
877
+ for i in range(len(video_grid_thw))
878
+ ]
879
+
880
+ return dict(
881
+ pixel_values=MultiModalFieldConfig.flat("image", image_slices),
882
+ image_embeds=MultiModalFieldConfig.flat("image", image_slices),
883
+ image_grid_thw=MultiModalFieldConfig.batched("image"),
884
+ pixel_values_videos=MultiModalFieldConfig.flat(
885
+ "video", video_slices),
886
+ video_embeds=MultiModalFieldConfig.flat("video", video_slices),
887
+ video_grid_thw=MultiModalFieldConfig.batched("video"),
888
+ )
889
+
890
def _get_dummy_mm_inputs(
891
self,
892
mm_counts: Mapping[str, int],
893
) -> ProcessorInputs:
851
- num_images = mm_counts["image"]
894
+ num_images = mm_counts.get("image", 0)
895
hf_processor = self._get_hf_processor()
896
image_token: str = hf_processor.image_token
897
image_processor = _get_image_processor(hf_processor)
@@ -869,7 +912,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
912
return ProcessorInputs(
913
prompt_text=image_token * num_images,
914
mm_data=data,
872
- mm_processor_kwargs={},
915
)
916
917
@@ -950,9 +992,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
992
return None
993
return quant_config
994
953
- def _validate_and_reshape_mm_tensor(self,
954
- mm_input: Union[torch.Tensor,
955
- List[torch.Tensor]],
995
+ def _validate_and_reshape_mm_tensor(self, mm_input: object,
996
name: str) -> torch.Tensor:
997
if not isinstance(mm_input, (torch.Tensor, list)):
998
raise ValueError(f"Incorrect type of {name}. "
@@ -962,7 +1002,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1002
return mm_input
1003
if mm_input.ndim != 3:
1004
raise ValueError(f"{name} should be 2D or batched 3D tensor. "
965
- f"Got ndim: {mm_input.ndim}")
1005
+ f"Got ndim: {mm_input.ndim} "
1006
+ f"(shape={mm_input.shape})")
1007
return torch.concat(list(mm_input))
1008
else:
1009
return torch.concat(mm_input)
vllm/model_executor/models/ultravox.py
+51
-25
@@ -23,10 +23,11 @@ from vllm.model_executor.layers.layernorm import RMSNorm
23
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24
from vllm.model_executor.model_loader.loader import DefaultModelLoader
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
-from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
26
+from vllm.multimodal import MULTIMODAL_REGISTRY
27
+from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
28
+ MultiModalKwargs, NestedTensors)
29
from vllm.multimodal.processing import (BaseMultiModalProcessor,
28
- MultiModalDataItems, ProcessorInputs,
29
- PromptReplacement)
30
+ ProcessorInputs, PromptReplacement)
31
from vllm.sequence import IntermediateTensors
32
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
33
from vllm.utils import is_list_of
@@ -72,11 +73,19 @@ def get_ultravox_max_audio_tokens(ctx: InputContext):
73
74
class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
75
76
+ def _get_hf_processor(
77
+ self,
78
+ *,
79
+ # Ignored in initialization
80
+ sampling_rate: Optional[int] = None,
81
+ ) -> ProcessorMixin:
82
+ return self.ctx.get_hf_processor()
83
+
84
def _get_feature_extractor(self) -> WhisperFeatureExtractor:
85
hf_processor = self._get_hf_processor()
86
return hf_processor.audio_processor.feature_extractor # type: ignore
87
79
- def _get_processor_data(
88
+ def _get_hf_mm_data(
89
self,
90
mm_items: MultiModalDataItems,
91
) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -84,33 +93,41 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
93
feature_extractor = self._get_feature_extractor()
94
mm_items.resample_audios(feature_extractor.sampling_rate)
95
87
- return super()._get_processor_data(mm_items)
96
+ return super()._get_hf_mm_data(mm_items)
97
98
def _call_hf_processor(
99
self,
91
- hf_processor: ProcessorMixin,
100
prompt: str,
93
- processor_data: Mapping[str, object],
94
- mm_processor_kwargs: Mapping[str, object],
101
+ mm_data: Mapping[str, object],
102
+ mm_kwargs: Mapping[str, object],
103
) -> BatchFeature:
96
- processor_data = dict(processor_data)
97
- audios = processor_data.pop("audios", [])
104
+ # Text-only input not supported in composite processor
105
+ if not mm_data:
106
+ tokenizer = self._get_tokenizer()
107
+
108
+ prompt_ids = tokenizer.encode(
109
+ prompt,
110
+ add_special_tokens=False, # type: ignore
111
+ )
112
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
113
+
114
+ mm_data = dict(mm_data)
115
+ audios = mm_data.pop("audios", [])
116
117
if not audios:
118
return super()._call_hf_processor(
101
- hf_processor,
119
prompt=prompt,
103
- processor_data=processor_data,
104
- mm_processor_kwargs=mm_processor_kwargs,
120
+ mm_data=mm_data,
121
+ mm_kwargs=mm_kwargs,
122
)
123
124
feature_extractor = self._get_feature_extractor()
108
- mm_processor_kwargs = dict(
109
- **mm_processor_kwargs,
125
+ mm_kwargs = dict(
126
+ **mm_kwargs,
127
sampling_rate=feature_extractor.sampling_rate,
128
)
129
113
- # Already resampled by _get_processor_data
130
+ # Already resampled by _get_hf_mm_data
131
assert is_list_of(audios, np.ndarray)
132
133
# Ultravox processor doesn't support multiple inputs,
@@ -119,13 +136,12 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
136
shared_outputs = {}
137
for audio in audios:
138
# NOTE: Ultravox processor accepts "audio" instead of "audios"
122
- item_processor_data = dict(**processor_data, audio=audio)
139
+ item_processor_data = dict(**mm_data, audio=audio)
140
141
item_outputs = super()._call_hf_processor(
125
- hf_processor,
142
prompt=prompt,
127
- processor_data=item_processor_data,
128
- mm_processor_kwargs=mm_processor_kwargs,
143
+ mm_data=item_processor_data,
144
+ mm_kwargs=mm_kwargs,
145
)
146
147
audio_features.append(item_outputs.pop("audio_values")[0])
@@ -139,17 +155,28 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
155
)
156
return BatchFeature(combined_outputs)
157
158
+ def _get_mm_fields_config(
159
+ self,
160
+ hf_inputs: BatchFeature,
161
+ hf_processor_mm_kwargs: Mapping[str, object],
162
+ ) -> Mapping[str, MultiModalFieldConfig]:
163
+ return dict(
164
+ audio_features=MultiModalFieldConfig.batched("audio"),
165
+ audio_token_len=MultiModalFieldConfig.batched("audio"),
166
+ audio_embeds=MultiModalFieldConfig.batched("audio"),
167
+ )
168
+
169
def _get_prompt_replacements(
170
self,
171
mm_items: MultiModalDataItems,
145
- hf_inputs: BatchFeature,
146
- mm_processor_kwargs: Mapping[str, object],
172
+ hf_processor_mm_kwargs: Mapping[str, object],
173
+ out_mm_kwargs: MultiModalKwargs,
174
) -> list[PromptReplacement]:
175
hf_processor = self._get_hf_processor()
176
placeholder = hf_processor.audio_token_replacement # type: ignore
177
178
def get_replacement_ultravox(item_idx: int):
152
- audio_token_len = hf_inputs["audio_token_len"][item_idx]
179
+ audio_token_len = out_mm_kwargs["audio_token_len"][item_idx]
180
return placeholder * audio_token_len
181
182
return [
@@ -168,14 +195,13 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
195
sampling_rate = feature_extractor.sampling_rate
196
audio_len = feature_extractor.chunk_length * sampling_rate
197
171
- audio_count = mm_counts["audio"]
198
+ audio_count = mm_counts.get("audio", 0)
199
audio = np.zeros(audio_len)
200
data = {"audio": [audio] * audio_count}
201
202
return ProcessorInputs(
203
prompt_text="<|audio|>" * audio_count,
204
mm_data=data,
178
- mm_processor_kwargs={},
205
)
206
207
vllm/multimodal/base.py
+23
-21
@@ -297,35 +297,37 @@ class MultiModalPlaceholderMap:
297
``MultiModalPlaceholderMap`` that relates the multi-modal embedding
298
vectors to their corresponding placeholders.
299
300
- Consider the following scenarios:
300
+ Examples:
301
302
- Prompt: |AAAA BBBB What's in these images?|
303
- Positions: |.................................|
302
+ .. code-block::
303
305
- images = [A, B]
306
- src_ranges = [(0, 4), (4, 8)]
307
- dest_ranges = [(0, 4), (5, 9)]
304
+ Prompt: |AAAA BBBB What's in these images?|
305
+ Positions: |.................................|
306
309
- Prompt: |AAAA BBBB What's in these images?|
310
- Positions: | ..... |
307
+ images = [A, B]
308
+ src_ranges = [(0, 4), (4, 8)]
309
+ dest_ranges = [(0, 4), (5, 9)]
310
312
- images = [A, B]
313
- src_ranges = [(2, 4), (4, 6)]
314
- dest_ranges = [(0, 2), (3, 5)]
311
+ Prompt: |AAAA BBBB What's in these images?|
312
+ Positions: | ..... |
313
316
- Prompt: |AAAA BBBB What's in these images?|
317
- Positions: | ......... |
314
+ images = [A, B]
315
+ src_ranges = [(2, 4), (4, 6)]
316
+ dest_ranges = [(0, 2), (3, 5)]
317
319
- images = [B]
320
- src_ranges = [(0, 4)]
321
- dest_ranges = [(0, 4)]
318
+ Prompt: |AAAA BBBB What's in these images?|
319
+ Positions: | ......... |
320
323
- Prompt: |AAAA BBBB What's in these images?|
324
- Positions: | .......................|
321
+ images = [B]
322
+ src_ranges = [(0, 4)]
323
+ dest_ranges = [(0, 4)]
324
326
- images = []
327
- src_ranges = []
328
- dest_ranges = []
325
+ Prompt: |AAAA BBBB What's in these images?|
326
+ Positions: | .......................|
327
+
328
+ images = []
329
+ src_ranges = []
330
+ dest_ranges = []
331
"""
332
seq_mm_data = seq_group.multi_modal_data
333
seq_mm_placeholders = seq_group.multi_modal_placeholders
vllm/multimodal/inputs.py
+423
-15
@@ -1,12 +1,16 @@
1
+from abc import ABC, abstractmethod
2
from collections import UserDict, defaultdict
2
-from typing import (Any, Dict, List, Literal, Mapping, Sequence, Tuple,
3
- TypedDict, TypeVar, Union, cast, final)
3
+from collections.abc import Mapping, Sequence
4
+from dataclasses import dataclass
5
+from typing import (Any, Literal, NamedTuple, TypedDict, TypeVar, Union, cast,
6
+ final)
7
8
import numpy as np
9
import torch
10
import torch.types
11
from PIL.Image import Image
9
-from typing_extensions import NotRequired, TypeAlias
12
+from transformers import BatchFeature
13
+from typing_extensions import NotRequired, TypeAlias, assert_never
14
15
from vllm.utils import JSONTree, is_list_of, json_map_leaves
16
@@ -44,7 +48,7 @@ item, which can be passed to a HuggingFace :code:`AudioProcessor`.
48
"""
49
# yapf: enable
50
47
-MultiModalData: TypeAlias = Union[_T, List[_T]]
51
+MultiModalData: TypeAlias = Union[_T, list[_T]]
52
"""
53
Either a single data item, or a list of data items.
54
@@ -79,13 +83,135 @@ Note:
83
"""
84
85
86
+class ImageSize(NamedTuple):
87
+ width: int
88
+ height: int
89
+
90
+
91
+class MultiModalDataItems(UserDict[str, list[Any]]):
92
+ """
93
+ As :class:`MultiModalDataDict`, but normalized such that each entry
94
+ corresponds to a list.
95
+ """
96
+
97
+ @staticmethod
98
+ def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
99
+ """
100
+ Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
101
+ """
102
+ multi_data = MultiModalDataItems()
103
+
104
+ for k, v in data.items():
105
+ # TODO: Make a separate modality for embedding inputs
106
+ # to avoid confusion
107
+ # yapf: disable
108
+ if k == "video":
109
+ # Special case since even a single item can be a list
110
+ multi_data[k] = ( # type: ignore[index]
111
+ v if (
112
+ isinstance(v, torch.Tensor)
113
+ or is_list_of(v, list)
114
+ or isinstance(v[0], (np.ndarray, torch.Tensor))
115
+ and v[0].ndim == 4
116
+ ) else [v]
117
+ )
118
+ elif k in ("image", "audio"):
119
+ multi_data[k] = ( # type: ignore[index]
120
+ v if isinstance(v, (torch.Tensor, list)) else [v]
121
+ )
122
+ else:
123
+ multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
124
+ # yapf: enable
125
+
126
+ return multi_data
127
+
128
+ # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
129
+ # `self.images` doesn't update this dictionary, which may be confusing
130
+ # We annotate the getter methods as `Sequence` to prevent others from
131
+ # trying to update the list in this way
132
+ @property
133
+ def images(self) -> Sequence[ImageItem]:
134
+ return self.get("image", [])
135
+
136
+ @property
137
+ def videos(self) -> Sequence[VideoItem]:
138
+ return self.get("video", [])
139
+
140
+ @property
141
+ def audios(self) -> Sequence[AudioItem]:
142
+ return self.get("audio", [])
143
+
144
+ def get_item_counts(self) -> Mapping[str, int]:
145
+ return {m: len(items) for m, items in self.items()}
146
+
147
+ def has_embedding_inputs(self) -> bool:
148
+ return any(
149
+ any(isinstance(item, torch.Tensor) for item in items)
150
+ for items in self.values())
151
+
152
+ def get_image_size(self, item_idx: int) -> ImageSize:
153
+ image = self.images[item_idx]
154
+
155
+ if isinstance(image, Image):
156
+ return ImageSize(*image.size)
157
+ if isinstance(image, (np.ndarray, torch.Tensor)):
158
+ _, h, w = image.shape
159
+ return ImageSize(w, h)
160
+
161
+ assert_never(image)
162
+
163
+ def get_audio_with_sr(
164
+ self,
165
+ item_idx: int,
166
+ *,
167
+ default_sr: float,
168
+ ) -> tuple[np.ndarray, float]:
169
+ audio = self.audios[item_idx]
170
+
171
+ if isinstance(audio, tuple):
172
+ return audio
173
+ if isinstance(audio, list):
174
+ return np.array(audio), default_sr
175
+ if isinstance(audio, np.ndarray):
176
+ return audio, default_sr
177
+
178
+ assert_never(audio)
179
+
180
+ def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
181
+ """
182
+ If :code:`drop_sr=True`, the audio items in this dictionary are updated
183
+ to be NumPy arrays which implicitly means that their sampling rate is
184
+ the same as the model's expected sampling rate; otherwise, they remain
185
+ as :code:`(audio, new_sr)` tuples.
186
+ """
187
+ # Avoid circular import
188
+ from .audio import resample_audio
189
+
190
+ if not self.audios:
191
+ return
192
+
193
+ new_audios = []
194
+ for item_idx in range(len(self.audios)):
195
+ audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
196
+ audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
197
+
198
+ new_audios.append(audio if drop_sr else (audio, new_sr))
199
+
200
+ self["audio"] = new_audios
201
+
202
+
203
class PlaceholderRange(TypedDict):
204
"""
205
Placeholder location information for multi-modal data.
206
86
- For example:
87
- Prompt: AAAA BBBB What is in these images?
207
+ Example:
208
+
209
+ Prompt: :code:`AAAA BBBB What is in these images?`
210
+
211
Images A and B will have:
212
+
213
+ .. code-block::
214
+
215
A: { "offset": 0, "length": 4 }
216
B: { "offset": 5, "length": 4 }
217
"""
@@ -97,25 +223,256 @@ class PlaceholderRange(TypedDict):
223
"""The length of the placeholder."""
224
225
100
-NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor,
101
- Tuple[torch.Tensor, ...]]
226
+NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,
227
+ tuple[torch.Tensor, ...]]
228
"""
229
Uses a list instead of a tensor if the dimensions of each element do not match.
230
"""
231
106
-BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
232
+
233
+def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
234
+ """Equality check between :data:`NestedTensors` objects."""
235
+ if isinstance(a, torch.Tensor):
236
+ return isinstance(b, torch.Tensor) and bool((a == b).all().item())
237
+ elif isinstance(b, torch.Tensor):
238
+ return isinstance(a, torch.Tensor) and bool((b == a).all().item())
239
+
240
+ if isinstance(a, list):
241
+ return (isinstance(b, list)
242
+ and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)))
243
+ if isinstance(b, list):
244
+ return (isinstance(a, list)
245
+ and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)))
246
+
247
+ # Both a and b are scalars
248
+ return a == b
249
+
250
+
251
+BatchedTensorInputs: TypeAlias = Mapping[str, NestedTensors]
252
"""
253
A dictionary containing nested tensors which have been batched via
254
:meth:`MultiModalKwargs.batch`.
255
"""
256
257
258
+@dataclass(frozen=True)
259
+class MultiModalFieldItem:
260
+ """
261
+ Contains metadata and data in :class:`MultiModalKwargs`
262
+ corresponding to a data item in :class:`MultiModalDataItems`.
263
+ """
264
+ field: "BaseMultiModalField"
265
+ data: NestedTensors
266
+
267
+ def __eq__(self, other: object) -> bool:
268
+ if not isinstance(other, self.__class__):
269
+ return False
270
+
271
+ return (self.field == other.field
272
+ and nested_tensors_equal(self.data, other.data))
273
+
274
+
275
+@dataclass(frozen=True)
276
+class BaseMultiModalField(ABC):
277
+ """Abstract base class for a field in :class:`MultiModalKwargs`."""
278
+ key: str
279
+ modality: str
280
+
281
+ @abstractmethod
282
+ def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
283
+ raise NotImplementedError
284
+
285
+ def _build_item(self, data: NestedTensors) -> MultiModalFieldItem:
286
+ return MultiModalFieldItem(self, data)
287
+
288
+ def reduce(self, batch: list[MultiModalFieldItem]) -> MultiModalFieldItem:
289
+ """Merge multiple instances of :class:`MultiModalFieldItem` together."""
290
+ fields = [item.field for item in batch]
291
+ if len(set(fields)) > 1:
292
+ raise ValueError(f"Cannot merge different {fields=}")
293
+
294
+ data = self._reduce_data([item.data for item in batch])
295
+
296
+ return self._build_item(data)
297
+
298
+
299
+@dataclass(frozen=True)
300
+class MultiModalBatchedField(BaseMultiModalField):
301
+ """
302
+ A :class:`BaseMultiModalField` implementation where an item is obtained by
303
+ directly indexing into the first dimension of the underlying data.
304
+ """
305
+
306
+ def build_items(self, batch: NestedTensors) -> list[MultiModalFieldItem]:
307
+ return [self._build_item(item) for item in batch]
308
+
309
+ def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
310
+ if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
311
+ first_shape = batch[0].shape
312
+ if all(item.shape == first_shape for item in batch):
313
+ return torch.stack(batch)
314
+
315
+ return batch
316
+
317
+
318
+@dataclass(frozen=True)
319
+class MultiModalFlatField(BaseMultiModalField):
320
+ """
321
+ A :class:`BaseMultiModalField` implementation where an item is obtained by
322
+ slicing along the first dimension of the underlying data.
323
+ """
324
+
325
+ def build_items(
326
+ self,
327
+ batch: NestedTensors,
328
+ slices: Sequence[slice],
329
+ ) -> list[MultiModalFieldItem]:
330
+ return [self._build_item(batch[slice_]) for slice_ in slices]
331
+
332
+ def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
333
+ if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
334
+ first_shape = batch[0].shape
335
+ if all(item.shape[1:] == first_shape[1:] for item in batch):
336
+ return torch.concat(batch)
337
+
338
+ return [elem for item in batch for elem in item]
339
+
340
+
341
+class MultiModalFieldConfig:
342
+
343
+ @staticmethod
344
+ def batched(modality: str):
345
+ return MultiModalFieldConfig(
346
+ field_cls=MultiModalBatchedField,
347
+ modality=modality,
348
+ )
349
+
350
+ @staticmethod
351
+ def flat(modality: str, slices: Sequence[slice]):
352
+ return MultiModalFieldConfig(
353
+ field_cls=MultiModalFlatField,
354
+ modality=modality,
355
+ slices=slices,
356
+ )
357
+
358
+ def __init__(
359
+ self,
360
+ field_cls: type[BaseMultiModalField],
361
+ modality: str,
362
+ **field_config: Any,
363
+ ) -> None:
364
+ super().__init__()
365
+
366
+ self._field_cls = field_cls
367
+ self._modality = modality
368
+ self._field_config = field_config
369
+
370
+ def build_items(
371
+ self,
372
+ key: str,
373
+ batch: NestedTensors,
374
+ ) -> list[MultiModalFieldItem]:
375
+ field = self._field_cls(key=key, modality=self._modality)
376
+ return field.build_items(batch, **self._field_config) # type: ignore
377
+
378
+
379
class MultiModalKwargs(UserDict[str, NestedTensors]):
380
"""
381
A dictionary that represents the keyword arguments to
382
:meth:`~torch.nn.Module.forward`.
383
+
384
+ The metadata :code:`items_by_key` defines how to split batched keyword
385
+ arguments corresponding to each data item in :class:`MultiModalDataItems`:
386
+
387
+ - For a keyword argument, we can access the :code:`i` th item in the batch
388
+ via :code:`items_by_key[key][i]`.
389
+ - We can gather the keyword arguments belonging to a modality by finding
390
+ the keys with items that belong to that modality, then accessing
391
+ the :code:`i` th item in the batch for each such key.
392
+
393
+ Example:
394
+
395
+ .. code-block:: python
396
+
397
+ # All items belong to the "image" modality
398
+ items_by_key={
399
+ "pixel_values": [a, b, c, d], # "image" modality
400
+ "image_grid_thw": [e, f, g, h], # "image" modality
401
+ "pixel_values_video": [h, i, j], # "video" modality
402
+ "video_grid_thw": [k, l, m], # "video" modality
403
+ }
404
+
405
+ - The keyword arguments belonging to the first image are
406
+ :code:`{"pixel_values": a, "image_grid_thw": e}`.
407
+ - The keyword arguments belonging to the second video are
408
+ :code:`{"pixel_values_video": i, "video_grid_thw": l}`.
409
"""
410
411
+ @staticmethod
412
+ def from_hf_inputs(
413
+ hf_inputs: BatchFeature,
414
+ config_by_key: Mapping[str, MultiModalFieldConfig],
415
+ *,
416
+ enable_sanity_checks: bool = False,
417
+ ):
418
+ # NOTE: This skips fields in `hf_inputs` that are not in `config_by_key`
419
+ # We assume that those fields are not used in vLLM
420
+ items_by_key = {
421
+ key: config.build_items(key, batch)
422
+ for key, config in config_by_key.items()
423
+ if (batch := hf_inputs.get(key)) is not None
424
+ }
425
+
426
+ return MultiModalKwargs.from_items_by_key(
427
+ items_by_key,
428
+ enable_sanity_checks=enable_sanity_checks,
429
+ )
430
+
431
+ @staticmethod
432
+ def from_items_by_key(
433
+ items_by_key: Mapping[str, list[MultiModalFieldItem]],
434
+ *,
435
+ enable_sanity_checks: bool = False,
436
+ ) -> "MultiModalKwargs":
437
+ data = {
438
+ key: items[0].field.reduce(items).data
439
+ for key, items in items_by_key.items()
440
+ }
441
+
442
+ return MultiModalKwargs(data,
443
+ items_by_key=items_by_key,
444
+ enable_sanity_checks=enable_sanity_checks)
445
+
446
+ def __init__(
447
+ self,
448
+ data: Mapping[str, NestedTensors],
449
+ *,
450
+ items_by_key: Mapping[str, list[MultiModalFieldItem]] = {},
451
+ enable_sanity_checks: bool = False,
452
+ ) -> None:
453
+ super().__init__(data)
454
+
455
+ # Shallow copy to avoid footgun in case a defaultdict is passed in
456
+ self._items_by_key = dict(items_by_key)
457
+
458
+ keys_by_modality = defaultdict[str, set[str]](set)
459
+ for key, items in items_by_key.items():
460
+ for item in items:
461
+ keys_by_modality[item.field.modality].add(key)
462
+
463
+ self._keys_by_modality = dict(keys_by_modality)
464
+
465
+ if enable_sanity_checks:
466
+ for modality, keys in keys_by_modality.items():
467
+ items_in_modality = {k: items_by_key[k] for k in keys}
468
+ batch_sizes = {k: len(v) for k, v in items_in_modality.items()}
469
+ batch_size = next(iter(batch_sizes.values()), 0)
470
+ assert all(bs == batch_size
471
+ for bs in batch_sizes.values()), dict(
472
+ modality=modality,
473
+ batch_sizes=batch_sizes,
474
+ items_by_key=items_by_key)
475
+
476
@staticmethod
477
def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
478
"""
@@ -139,7 +496,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
496
# Only tensors (not lists) can be stacked.
497
return stacked
498
142
- tensors_ = cast(List[torch.Tensor], stacked)
499
+ tensors_ = cast(list[torch.Tensor], stacked)
500
if any(t.shape != tensors_[0].shape for t in tensors_):
501
# The tensors have incompatible shapes and can't be stacked.
502
return tensors_
@@ -147,7 +504,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
504
return torch.stack(tensors_)
505
506
@staticmethod
150
- def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
507
+ def batch(inputs_list: list["MultiModalKwargs"]) -> BatchedTensorInputs:
508
"""
509
Batch multiple inputs together into a dictionary.
510
@@ -162,7 +519,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
519
520
# We need to consider the case where each item in the batch
521
# contains different modalities (i.e. different keys).
165
- item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
522
+ item_lists = defaultdict[str, list[NestedTensors]](list)
523
524
for inputs in inputs_list:
525
for k, v in inputs.items():
@@ -188,6 +545,57 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
545
546
return cast(BatchedTensorInputs, json_mapped)
547
548
+ def __eq__(self, other: object) -> bool:
549
+ if not isinstance(other, self.__class__):
550
+ return False
551
+ if self._items_by_key != other._items_by_key:
552
+ return False
553
+
554
+ ks = self.keys()
555
+ return (ks == other.keys()
556
+ and all(nested_tensors_equal(self[k], other[k]) for k in ks))
557
+
558
+ def get_item(self, key: str, item_index: int) -> MultiModalFieldItem:
559
+ return self._items_by_key[key][item_index]
560
+
561
+ def get_items_by_modality(
562
+ self,
563
+ modality: str,
564
+ item_index: int,
565
+ ) -> Mapping[str, MultiModalFieldItem]:
566
+ """
567
+ Get the keyword arguments corresponding to an item identified by
568
+ its modality and index.
569
+ """
570
+ keys_to_gather = self._keys_by_modality[modality]
571
+
572
+ return {
573
+ key: self.get_item(key, item_index)
574
+ for key in keys_to_gather if key in self
575
+ }
576
+
577
+ @staticmethod
578
+ def from_items_by_modality(
579
+ items_by_modality: Mapping[str, list[Mapping[str,
580
+ MultiModalFieldItem]]],
581
+ *,
582
+ enable_sanity_checks: bool = False,
583
+ ) -> "MultiModalKwargs":
584
+ """
585
+ Construct a new :class:`MultiModalKwargs` from multiple items returned
586
+ by :meth:`get_fields_by_modality`.
587
+ """
588
+ items_by_key = defaultdict[str, list[MultiModalFieldItem]](list)
589
+ for fields in items_by_modality.values():
590
+ for field in fields:
591
+ for k, v in field.items():
592
+ items_by_key[k].append(v)
593
+
594
+ return MultiModalKwargs.from_items_by_key(
595
+ items_by_key,
596
+ enable_sanity_checks=enable_sanity_checks,
597
+ )
598
+
599
600
MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]
601
"""
@@ -207,16 +615,16 @@ class MultiModalInputsV2(TypedDict):
615
prompt: str
616
"""The processed prompt text."""
617
210
- prompt_token_ids: List[int]
618
+ prompt_token_ids: list[int]
619
"""The processed token IDs which includes placeholder tokens."""
620
213
- token_type_ids: NotRequired[List[int]]
621
+ token_type_ids: NotRequired[list[int]]
622
"""The token type IDs of the prompt."""
623
624
mm_kwargs: MultiModalKwargs
625
"""Keyword arguments to be directly passed to the model after batching."""
626
219
- mm_hashes: NotRequired[List[str]]
627
+ mm_hashes: NotRequired[list[str]]
628
"""The hashes of the multi-modal data."""
629
630
mm_placeholders: MultiModalPlaceholderDict
vllm/multimodal/processing.py
+347
-169
@@ -1,6 +1,6 @@
1
+import pickle
2
import re
3
from abc import ABC, abstractmethod
3
-from collections import UserDict
4
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
5
from dataclasses import dataclass, field
6
from functools import lru_cache
@@ -8,19 +8,18 @@ from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
8
9
import numpy as np
10
import torch
11
+from blake3 import blake3
12
from PIL.Image import Image
13
from transformers import BatchFeature, ProcessorMixin
13
-from typing_extensions import assert_never
14
15
from vllm.inputs import DummyData, InputProcessingContext
16
from vllm.logger import init_logger
17
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18
-from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
18
+from vllm.utils import LRUCache, flatten_2d_lists, full_groupby, is_list_of
19
20
-from .audio import resample_audio
21
-from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
22
- MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
23
- VideoItem)
20
+from .inputs import (MultiModalDataDict, MultiModalDataItems,
21
+ MultiModalFieldConfig, MultiModalFieldItem,
22
+ MultiModalInputsV2, MultiModalKwargs, PlaceholderRange)
23
24
logger = init_logger(__name__)
25
@@ -201,111 +200,6 @@ class _BoundPromptReplacement:
200
return bound_replacement
201
202
204
-class ImageSize(NamedTuple):
205
- width: int
206
- height: int
207
-
208
-
209
-class MultiModalDataItems(UserDict[str, list[Any]]):
210
- """
211
- As :class:`MultiModalDataDict`, but normalized such that each entry
212
- corresponds to a list.
213
- """
214
-
215
- @staticmethod
216
- def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
217
- """
218
- Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
219
- """
220
- multi_data = MultiModalDataItems()
221
-
222
- for k, v in data.items():
223
- # TODO: Make a separate modality for embedding inputs
224
- # to avoid confusion
225
- # yapf: disable
226
- if k == "video":
227
- # Special case since even a single item can be a list
228
- multi_data[k] = ( # type: ignore[index]
229
- v if (isinstance(v, torch.Tensor)
230
- or is_list_of(v, list)) else [v]
231
- )
232
- elif k in ("image", "audio"):
233
- multi_data[k] = ( # type: ignore[index]
234
- v if isinstance(v, (torch.Tensor, list)) else [v]
235
- )
236
- else:
237
- multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
238
- # yapf: enable
239
-
240
- return multi_data
241
-
242
- # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
243
- # `self.images` doesn't update this dictionary, which may be confusing
244
- # We annotate the getter methods as `Sequence` to prevent others from
245
- # trying to update the list in this way
246
- @property
247
- def images(self) -> Sequence[ImageItem]:
248
- return self.get("image", [])
249
-
250
- @property
251
- def videos(self) -> Sequence[VideoItem]:
252
- return self.get("video", [])
253
-
254
- @property
255
- def audios(self) -> Sequence[AudioItem]:
256
- return self.get("audio", [])
257
-
258
- def get_item_counts(self) -> Mapping[str, int]:
259
- return {m: len(items) for m, items in self.items()}
260
-
261
- def get_image_size(self, item_idx: int) -> ImageSize:
262
- image = self.images[item_idx]
263
-
264
- if isinstance(image, Image):
265
- return ImageSize(*image.size)
266
- if isinstance(image, (np.ndarray, torch.Tensor)):
267
- _, h, w = image.shape
268
- return ImageSize(w, h)
269
-
270
- assert_never(image)
271
-
272
- def get_audio_with_sr(
273
- self,
274
- item_idx: int,
275
- *,
276
- default_sr: float,
277
- ) -> tuple[np.ndarray, float]:
278
- audio = self.audios[item_idx]
279
-
280
- if isinstance(audio, tuple):
281
- return audio
282
- if isinstance(audio, list):
283
- return np.array(audio), default_sr
284
- if isinstance(audio, np.ndarray):
285
- return audio, default_sr
286
-
287
- assert_never(audio)
288
-
289
- def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
290
- """
291
- If :code:`drop_sr=True`, the audio items in this dictionary are updated
292
- to be NumPy arrays which implicitly means that their sampling rate is
293
- the same as the model's expected sampling rate; otherwise, they remain
294
- as :code:`(audio, new_sr)` tuples.
295
- """
296
- if not self.audios:
297
- return
298
-
299
- new_audios = []
300
- for item_idx in range(len(self.audios)):
301
- audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
302
- audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
303
-
304
- new_audios.append(audio if drop_sr else (audio, new_sr))
305
-
306
- self["audio"] = new_audios
307
-
308
-
203
class _TokenMatch(NamedTuple):
204
start_idx: int
205
end_idx: int
@@ -583,11 +477,124 @@ def iter_placeholders(
477
)
478
479
586
-class ProcessorInputs(NamedTuple):
587
- """Keyword arguments to :meth:`BaseMultiModalProcessor`"""
480
+@dataclass
481
+class ProcessorInputs:
482
+ """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
483
prompt_text: str
484
mm_data: MultiModalDataDict
590
- mm_processor_kwargs: Mapping[str, object]
485
+ hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
486
+
487
+
488
+class ProcessingCache:
489
+
490
+ def __init__(self, capacity: int) -> None:
491
+ super().__init__()
492
+
493
+ # DEBUG: Set to None to disable
494
+ self.debug_cache_hit_ratio_steps: Optional[int] = None
495
+
496
+ self._cache = LRUCache[str, Mapping[str,
497
+ MultiModalFieldItem]](capacity)
498
+
499
+ def _maybe_log_cache_stats(self) -> None:
500
+ steps = self.debug_cache_hit_ratio_steps
501
+ if not steps:
502
+ return
503
+
504
+ cache_stats = self._cache.stat()
505
+ if cache_stats.total % steps == 0:
506
+ logger.debug("ProcessingCache: hit_ratio = %.2f",
507
+ cache_stats.hit_ratio)
508
+
509
+ def _serialize_item(self, obj: object) -> bytes:
510
+ # Simple cases
511
+ if isinstance(obj, str):
512
+ return obj.encode("utf-8")
513
+ if isinstance(obj, bytes):
514
+ return obj
515
+ if isinstance(obj, Image):
516
+ return obj.tobytes()
517
+
518
+ # Convertible to NumPy arrays
519
+ if isinstance(obj, torch.Tensor):
520
+ obj = obj.numpy()
521
+ if isinstance(obj, (int, float)):
522
+ obj = np.array(obj)
523
+ if isinstance(obj, np.ndarray):
524
+ return obj.tobytes()
525
+
526
+ logger.warning(
527
+ "No serialization method found for %s. "
528
+ "Falling back to pickle.", type(obj))
529
+
530
+ return pickle.dumps(obj)
531
+
532
+ def _item_to_bytes(
533
+ self,
534
+ key: str,
535
+ obj: object,
536
+ ) -> Iterable[tuple[bytes, bytes]]:
537
+ # Recursive cases
538
+ if isinstance(obj, (list, tuple)):
539
+ for i, elem in enumerate(obj):
540
+ yield from self._item_to_bytes(f"{key}.{i}", elem)
541
+ elif isinstance(obj, dict):
542
+ for k, v in obj.items():
543
+ yield from self._item_to_bytes(f"{key}.{k}", v)
544
+ else:
545
+ key_bytes = self._serialize_item(key)
546
+ value_bytes = self._serialize_item(obj)
547
+ yield key_bytes, value_bytes
548
+
549
+ def _hash_kwargs(self, **kwargs: object) -> str:
550
+ hasher = blake3()
551
+
552
+ for k, v in kwargs.items():
553
+ for k_bytes, v_bytes in self._item_to_bytes(k, v):
554
+ hasher.update(k_bytes)
555
+ hasher.update(v_bytes)
556
+
557
+ return hasher.hexdigest()
558
+
559
+ def get(
560
+ self,
561
+ model_id: str,
562
+ modality: str,
563
+ input_item: object,
564
+ input_kwargs: Mapping[str, object],
565
+ ) -> Optional[Mapping[str, MultiModalFieldItem]]:
566
+ """
567
+ Get a processed multi-modal item from the cache
568
+ according to its dependencies, including:
569
+
570
+ - The model ID
571
+ - The modality of the item
572
+ - The original data item passed to the HF processor
573
+ - The configuration options of the HF processor
574
+ """
575
+ self._maybe_log_cache_stats()
576
+
577
+ cache_key = self._hash_kwargs(model_id=model_id,
578
+ **{modality: input_item},
579
+ **input_kwargs)
580
+ return self._cache.get(cache_key)
581
+
582
+ def put(
583
+ self,
584
+ model_id: str,
585
+ modality: str,
586
+ input_item: object,
587
+ input_kwargs: Mapping[str, object],
588
+ output_kwargs: Mapping[str, MultiModalFieldItem],
589
+ ) -> None:
590
+ """
591
+ Put a processed multi-modal item into the cache
592
+ according to its dependencies (see :meth:`get`).
593
+ """
594
+ cache_key = self._hash_kwargs(model_id=model_id,
595
+ **{modality: input_item},
596
+ **input_kwargs)
597
+ self._cache.put(cache_key, output_kwargs)
598
599
600
class BaseMultiModalProcessor(ABC):
@@ -595,18 +602,24 @@ class BaseMultiModalProcessor(ABC):
602
Abstract base class to process multi-modal inputs to be used in vLLM.
603
"""
604
598
- def __init__(self, ctx: InputProcessingContext) -> None:
605
+ def __init__(self,
606
+ ctx: InputProcessingContext,
607
+ *,
608
+ cache: Optional[ProcessingCache] = None,
609
+ enable_sanity_checks: bool = True) -> None:
610
super().__init__()
611
612
self.ctx = ctx
613
+ self.cache = cache
614
+ self.enable_sanity_checks = enable_sanity_checks
615
616
def __call__(
617
self,
618
prompt: str,
619
mm_data: MultiModalDataDict,
607
- mm_processor_kwargs: Mapping[str, object],
620
+ hf_processor_mm_kwargs: Mapping[str, object],
621
) -> MultiModalInputsV2:
609
- return self.apply(prompt, mm_data, mm_processor_kwargs)
622
+ return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
623
624
def _get_hf_processor(self) -> ProcessorMixin:
625
"""
@@ -624,12 +637,21 @@ class BaseMultiModalProcessor(ABC):
637
) -> MultiModalDataItems:
638
return MultiModalDataItems.from_dict(mm_data)
639
640
+ @abstractmethod
641
+ def _get_mm_fields_config(
642
+ self,
643
+ hf_inputs: BatchFeature,
644
+ hf_processor_mm_kwargs: Mapping[str, object],
645
+ ) -> Mapping[str, MultiModalFieldConfig]:
646
+ """Given the HF-processed data, output the metadata of each field."""
647
+ raise NotImplementedError
648
+
649
@abstractmethod
650
def _get_prompt_replacements(
651
self,
652
mm_items: MultiModalDataItems,
631
- hf_inputs: BatchFeature,
632
- mm_processor_kwargs: Mapping[str, object],
653
+ hf_processor_mm_kwargs: Mapping[str, object],
654
+ out_mm_kwargs: MultiModalKwargs,
655
) -> list[PromptReplacement]:
656
"""
657
Given the original multi-modal items for this modality
@@ -651,7 +673,7 @@ class BaseMultiModalProcessor(ABC):
673
return list(
674
iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
675
654
- def _get_processor_data(
676
+ def _get_hf_mm_data(
677
self,
678
mm_items: MultiModalDataItems,
679
) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -669,7 +691,7 @@ class BaseMultiModalProcessor(ABC):
691
and v[0].ndim == 2):
692
# Pass through embedding inputs (multi)
693
passthrough_data[f"{k}_embeds"] = v
672
- else:
694
+ elif len(v) > 0:
695
# Map keys to plural form, e.g.: image -> images
696
processor_data[f"{k}s"] = v
697
else:
@@ -679,39 +701,181 @@ class BaseMultiModalProcessor(ABC):
701
702
def _call_hf_processor(
703
self,
682
- hf_processor: ProcessorMixin,
704
prompt: str,
684
- processor_data: Mapping[str, object],
685
- mm_processor_kwargs: Mapping[str, object],
705
+ # Not to be confused with `mm_data` in `self.apply`.
706
+ # This refers to the data to be passed to HF processor.
707
+ mm_data: Mapping[str, object],
708
+ mm_kwargs: Mapping[str, object],
709
) -> BatchFeature:
710
return self.ctx.call_hf_processor(
688
- hf_processor,
689
- prompt,
690
- processor_data,
691
- mm_processor_kwargs,
711
+ self._get_hf_processor(**mm_kwargs),
712
+ dict(text=prompt, **mm_data),
713
+ mm_kwargs,
714
)
715
716
def _apply_hf_processor(
717
self,
696
- prompt: str,
718
+ prompt_text: str,
719
mm_items: MultiModalDataItems,
698
- mm_processor_kwargs: Mapping[str, object],
699
- ) -> BatchFeature:
700
- # some mm_processor_kwargs may be used in processor initialization
701
- # instead of processor call
702
- hf_processor = self._get_hf_processor(**mm_processor_kwargs)
720
+ hf_processor_mm_kwargs: Mapping[str, object],
721
+ ) -> tuple[list[int], MultiModalKwargs]:
722
+ """
723
+ Apply the HF processor on the full prompt text and multi-modal data.
724
+ """
725
+ processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
726
+
727
+ processed_data = self._call_hf_processor(
728
+ prompt=prompt_text,
729
+ mm_data=processor_data,
730
+ mm_kwargs=hf_processor_mm_kwargs,
731
+ )
732
+ processed_data.update(passthrough_data)
733
704
- processor_data, passthrough_data = self._get_processor_data(mm_items)
734
+ prompt_ids, = processed_data.pop("input_ids").tolist()
735
706
- hf_inputs = self._call_hf_processor(
707
- hf_processor,
708
- prompt=prompt,
709
- processor_data=processor_data,
710
- mm_processor_kwargs=mm_processor_kwargs,
736
+ mm_kwargs = MultiModalKwargs.from_hf_inputs(
737
+ processed_data,
738
+ self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
739
+ enable_sanity_checks=self.enable_sanity_checks,
740
)
712
- hf_inputs.update(passthrough_data)
741
714
- return hf_inputs
742
+ return prompt_ids, mm_kwargs
743
+
744
+ def _apply_hf_processor_missing(
745
+ self,
746
+ prompt_text: str,
747
+ mm_missing_data_items: MultiModalDataItems,
748
+ hf_processor_mm_kwargs: Mapping[str, object],
749
+ ):
750
+ """
751
+ Apply the HF processor on the full prompt text, but only on the
752
+ multi-modal data that are missing from the cache.
753
+
754
+ Note: We pass prompt text and multi-modal data into the HF processor
755
+ in separate calls to avoid HF prompt replacement being done for
756
+ cached items; instead, we rely on our own prompt replacement logic
757
+ for the full text.
758
+ """
759
+ mm_missing_counts = mm_missing_data_items.get_item_counts()
760
+
761
+ prompt_ids, _ = self._apply_hf_processor(
762
+ prompt_text=prompt_text,
763
+ mm_items=MultiModalDataItems({}),
764
+ hf_processor_mm_kwargs={},
765
+ )
766
+
767
+ # Some HF processors (e.g. Qwen2-VL) expect corresponding
768
+ # multi-modal tokens to be in the prompt text
769
+ dummy_inputs = self._get_dummy_mm_inputs(mm_missing_counts)
770
+
771
+ _, mm_missing_kwargs = self._apply_hf_processor(
772
+ prompt_text=dummy_inputs.prompt_text,
773
+ mm_items=mm_missing_data_items,
774
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
775
+ )
776
+
777
+ return prompt_ids, mm_missing_kwargs
778
+
779
+ def _cached_apply_hf_processor(
780
+ self,
781
+ prompt_text: str,
782
+ mm_data_items: MultiModalDataItems,
783
+ hf_processor_mm_kwargs: Mapping[str, object],
784
+ ) -> tuple[list[int], MultiModalKwargs]:
785
+ """
786
+ Apply the HF processor on the full prompt text,
787
+ caching the results and reusing cached results.
788
+ """
789
+ cache = self.cache
790
+ model_id = self.ctx.model_config.model
791
+
792
+ if cache is None or mm_data_items.has_embedding_inputs():
793
+ return self._apply_hf_processor(
794
+ prompt_text=prompt_text,
795
+ mm_items=mm_data_items,
796
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
797
+ )
798
+
799
+ mm_maybe_cached_field_items = {
800
+ modality: [
801
+ cache.get(model_id, modality, item, hf_processor_mm_kwargs)
802
+ for item in items
803
+ ]
804
+ for modality, items in mm_data_items.items()
805
+ }
806
+
807
+ mm_missing_idxs = {
808
+ modality: [idx for idx, out in enumerate(fields) if out is None]
809
+ for modality, fields in mm_maybe_cached_field_items.items()
810
+ }
811
+ mm_missing_data = {
812
+ modality: [mm_data_items[modality][idx] for idx in idxs]
813
+ for modality, idxs in mm_missing_idxs.items()
814
+ }
815
+ mm_missing_data_items = self._get_mm_items(mm_missing_data)
816
+
817
+ prompt_ids, mm_missing_kwargs = self._apply_hf_processor_missing(
818
+ prompt_text=prompt_text,
819
+ mm_missing_data_items=mm_missing_data_items,
820
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
821
+ )
822
+
823
+ mm_missing_next_idx = {
824
+ modality: 0
825
+ for modality in mm_missing_data_items
826
+ }
827
+
828
+ mm_merged_field_items = dict[str, list[Mapping[str,
829
+ MultiModalFieldItem]]]()
830
+ for modality, modal_items_lst in mm_maybe_cached_field_items.items():
831
+ merged_modal_items_lst = list[Mapping[str, MultiModalFieldItem]]()
832
+
833
+ for idx, modal_items in enumerate(modal_items_lst):
834
+ if modal_items is None:
835
+ modal_items = mm_missing_kwargs.get_items_by_modality(
836
+ modality,
837
+ mm_missing_next_idx[modality],
838
+ )
839
+
840
+ cache.put(
841
+ model_id,
842
+ modality,
843
+ mm_data_items[modality][idx],
844
+ hf_processor_mm_kwargs,
845
+ modal_items,
846
+ )
847
+
848
+ mm_missing_next_idx[modality] += 1
849
+
850
+ merged_modal_items_lst.append(modal_items)
851
+
852
+ mm_merged_field_items[modality] = merged_modal_items_lst
853
+
854
+ if self.enable_sanity_checks:
855
+ mm_missing_counts = mm_missing_data_items.get_item_counts()
856
+ assert all(
857
+ item_count == mm_missing_counts[modality]
858
+ for modality, item_count in mm_missing_next_idx.items()), dict(
859
+ mm_missing_next_idx=mm_missing_next_idx,
860
+ mm_missing_counts=mm_missing_counts)
861
+
862
+ mm_kwargs = MultiModalKwargs.from_items_by_modality(
863
+ mm_merged_field_items,
864
+ enable_sanity_checks=self.enable_sanity_checks,
865
+ )
866
+
867
+ if self.enable_sanity_checks:
868
+ mm_item_counts = mm_data_items.get_item_counts()
869
+
870
+ for modality, item_count in mm_item_counts.items():
871
+ for item_idx in range(item_count):
872
+ try:
873
+ mm_kwargs.get_items_by_modality(modality, item_idx)
874
+ except Exception as e:
875
+ # Make it easy to set a breakpoint in the debugger
876
+ raise e
877
+
878
+ return prompt_ids, mm_kwargs
879
880
def _bind_prompt_replacements(
881
self,
@@ -730,6 +894,10 @@ class BaseMultiModalProcessor(ABC):
894
tokenizer = self._get_tokenizer()
895
896
token_matches = find_token_matches(token_ids, prompt_repls)
897
+ mm_match_counts = {
898
+ modality: len(matches)
899
+ for modality, matches in full_groupby_modality(token_matches)
900
+ }
901
902
# If the search text does not represent a special token,
903
# it may have different token IDs in the prompt, because
@@ -742,8 +910,8 @@ class BaseMultiModalProcessor(ABC):
910
# of the search text in the prompt, we instead perform string
911
# replacement on the decoded token IDs, then encode them back.
912
if all(
745
- len(matches) >= mm_item_counts[modality]
746
- for modality, matches in full_groupby_modality(token_matches)
913
+ mm_match_counts.get(modality, 0) >= item_count
914
+ for modality, item_count in mm_item_counts.items()
915
): # yapf: disable
916
token_ids = replace_token_matches(
917
token_ids,
@@ -775,7 +943,7 @@ class BaseMultiModalProcessor(ABC):
943
self,
944
prompt_text: str,
945
mm_data: MultiModalDataDict,
778
- mm_processor_kwargs: Mapping[str, object],
946
+ hf_processor_mm_kwargs: Mapping[str, object],
947
) -> MultiModalInputsV2:
948
"""
949
Process multi-modal inputs to be used in vLLM.
@@ -792,20 +960,24 @@ class BaseMultiModalProcessor(ABC):
960
"""
961
mm_items = self._get_mm_items(mm_data)
962
795
- hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
796
- mm_processor_kwargs)
797
- prompt_ids, = hf_inputs.pop("input_ids").tolist()
798
- mm_kwargs = MultiModalKwargs(hf_inputs)
963
+ prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
964
+ prompt_text,
965
+ mm_items,
966
+ hf_processor_mm_kwargs,
967
+ )
968
800
- prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
801
- mm_processor_kwargs)
802
- all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
969
+ unbound_prompt_repls = self._get_prompt_replacements(
970
+ mm_items,
971
+ hf_processor_mm_kwargs,
972
+ mm_kwargs,
973
+ )
974
+ prompt_repls = self._bind_prompt_replacements(unbound_prompt_repls)
975
976
# If HF processor already inserts placeholder tokens,
977
# there is no need for us to insert them
978
mm_item_counts = mm_items.get_item_counts()
807
- all_placeholders = self._find_placeholders(all_prompt_repls,
808
- prompt_ids, mm_item_counts)
979
+ all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
980
+ mm_item_counts)
981
982
if all_placeholders:
983
tokenizer = self._get_tokenizer()
@@ -817,7 +989,7 @@ class BaseMultiModalProcessor(ABC):
989
all_placeholders,
990
) = self._apply_prompt_replacements(
991
prompt_ids,
820
- all_prompt_repls,
992
+ prompt_repls,
993
mm_item_counts,
994
)
995
@@ -855,23 +1027,29 @@ class BaseMultiModalProcessor(ABC):
1027
from vllm.sequence import SequenceData
1028
1029
processor_inputs = self._get_dummy_mm_inputs(mm_counts)
858
- mm_inputs = self.apply(*processor_inputs)
1030
+ mm_inputs = self.apply(
1031
+ prompt_text=processor_inputs.prompt_text,
1032
+ mm_data=processor_inputs.mm_data,
1033
+ hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1034
+ )
1035
1036
prompt_token_ids = mm_inputs["prompt_token_ids"]
1037
placeholders_by_modality = mm_inputs["mm_placeholders"]
1038
863
- total_placeholders_by_modality = dict[str, int]()
864
- for modality, placeholders in placeholders_by_modality.items():
865
- num_placeholders = sum(item["length"] for item in placeholders)
866
- max_tokens = mm_max_tokens[modality]
867
-
868
- if num_placeholders != max_tokens:
869
- logger.warning(
870
- "The processed dummy data has a total of %d placeholder "
871
- "tokens for the '%s' modality, which is not the expected "
872
- "%d tokens.", num_placeholders, modality, max_tokens)
873
-
874
- total_placeholders_by_modality[modality] = num_placeholders
1039
+ total_placeholders_by_modality = {
1040
+ modality: sum(item["length"] for item in placeholders)
1041
+ for modality, placeholders in placeholders_by_modality.items()
1042
+ }
1043
+ expected_placeholders_by_modality = {
1044
+ modality: mm_max_tokens[modality]
1045
+ for modality in placeholders_by_modality
1046
+ }
1047
+ if total_placeholders_by_modality != expected_placeholders_by_modality:
1048
+ raise AssertionError(
1049
+ f"The processed dummy data has a total of "
1050
+ f"{total_placeholders_by_modality} placeholder tokens, which "
1051
+ f"is not the expected {expected_placeholders_by_modality} "
1052
+ "tokens.")
1053
1054
total_len = len(prompt_token_ids)
1055
if total_len > seq_len:
vllm/multimodal/registry.py
+30
-20
@@ -1,10 +1,9 @@
1
import functools
2
from collections import UserDict
3
-from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
3
+from typing import (TYPE_CHECKING, Any, Dict, Mapping, Optional, Protocol,
4
Sequence, Type, TypeVar)
5
6
import torch.nn as nn
7
-from typing_extensions import TypeAlias
7
8
from vllm.inputs import InputProcessingContext
9
from vllm.logger import init_logger
@@ -15,7 +14,7 @@ from .audio import AudioPlugin
14
from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
15
from .image import ImagePlugin
16
from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18
-from .processing import BaseMultiModalProcessor
17
+from .processing import BaseMultiModalProcessor, ProcessingCache
18
from .video import VideoPlugin
19
20
if TYPE_CHECKING:
@@ -23,15 +22,22 @@ if TYPE_CHECKING:
22
23
logger = init_logger(__name__)
24
25
+# TODO: Tune the MM cache size
26
+MM_CACHE_SIZE = 256
27
+
28
N = TypeVar("N", bound=Type[nn.Module])
29
28
-MultiModalProcessorFactory: TypeAlias = Callable[[InputProcessingContext],
29
- BaseMultiModalProcessor]
30
-"""
31
-Constructs a :class:`MultiModalProcessor` instance from the context.
30
33
-The processing metadata should be derived from the context.
34
-"""
31
+class MultiModalProcessorFactory(Protocol):
32
+ """Constructs a :class:`MultiModalProcessor` instance from the context."""
33
+
34
+ def __call__(
35
+ self,
36
+ ctx: InputProcessingContext,
37
+ *,
38
+ cache: Optional[ProcessingCache] = None,
39
+ ) -> BaseMultiModalProcessor:
40
+ ...
41
42
43
class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
@@ -71,6 +77,8 @@ class MultiModalRegistry:
77
78
self._limits_by_model = _MultiModalLimits()
79
80
+ self._processing_cache = ProcessingCache(MM_CACHE_SIZE)
81
+
82
def register_plugin(self, plugin: MultiModalPlugin) -> None:
83
"""
84
Register a multi-modal plugin so it can be recognized by vLLM.
@@ -328,15 +336,18 @@ class MultiModalRegistry:
336
337
return wrapper
338
331
- def has_processor(self, model_config: "ModelConfig") -> bool:
332
- """
333
- Test whether a multi-modal processor is defined for a specific model.
334
- """
339
+ def _get_model_cls(self, model_config: "ModelConfig"):
340
# Avoid circular import
341
from vllm.model_executor.model_loader import get_model_architecture
342
343
model_cls, _ = get_model_architecture(model_config)
339
- return model_cls in self._processor_factories
344
+ return model_cls
345
+
346
+ def has_processor(self, model_config: "ModelConfig") -> bool:
347
+ """
348
+ Test whether a multi-modal processor is defined for a specific model.
349
+ """
350
+ return self._get_model_cls(model_config) in self._processor_factories
351
352
def create_processor(
353
self,
@@ -346,12 +357,11 @@ class MultiModalRegistry:
357
"""
358
Create a multi-modal processor for a specific model and tokenizer.
359
"""
349
-
350
- # Avoid circular import
351
- from vllm.model_executor.model_loader import get_model_architecture
352
-
353
- model_cls, _ = get_model_architecture(model_config)
360
+ model_cls = self._get_model_cls(model_config)
361
processor_factory = self._processor_factories[model_cls]
362
363
ctx = InputProcessingContext(model_config, tokenizer)
357
- return processor_factory(ctx)
364
+ cache = (None if model_config.disable_mm_preprocessor_cache else
365
+ self._processing_cache)
366
+
367
+ return processor_factory(ctx, cache=cache)
vllm/transformers_utils/processor.py
+9
-3
@@ -1,25 +1,31 @@
1
from functools import lru_cache
2
from typing import Any, cast
3
4
+from transformers.processing_utils import ProcessorMixin
5
+
6
7
def get_processor(
8
processor_name: str,
9
*args: Any,
10
trust_remote_code: bool = False,
11
+ processor_cls: type[ProcessorMixin] = ProcessorMixin,
12
**kwargs: Any,
13
):
14
"""Load a processor for the given model name via HuggingFace."""
15
# don't put this import at the top level
16
# it will call torch.cuda.device_count()
17
from transformers import AutoProcessor
15
- from transformers.processing_utils import ProcessorMixin
18
+
19
+ processor_factory = (AutoProcessor
20
+ if processor_cls == ProcessorMixin else processor_cls)
21
22
try:
18
- processor = AutoProcessor.from_pretrained(
23
+ processor = processor_factory.from_pretrained(
24
processor_name,
25
*args,
26
trust_remote_code=trust_remote_code,
22
- **kwargs)
27
+ **kwargs,
28
+ )
29
except ValueError as e:
30
# If the error pertains to the processor class not existing or not
31
# currently being imported, suggest using the --trust-remote-code flag.
vllm/utils.py
+25
-2
@@ -25,11 +25,11 @@ import warnings
25
import weakref
26
from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
27
from collections import OrderedDict, UserDict, defaultdict
28
-from collections.abc import Iterable, Mapping
28
+from collections.abc import Hashable, Iterable, Mapping
29
from dataclasses import dataclass, field
30
from functools import lru_cache, partial, wraps
31
from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
32
- Dict, Generator, Generic, Hashable, List, Literal,
32
+ Dict, Generator, Generic, List, Literal, NamedTuple,
33
Optional, Tuple, Type, TypeVar, Union, overload)
34
from uuid import uuid4
35
@@ -194,13 +194,29 @@ class Counter:
194
self.counter = 0
195
196
197
+class CacheInfo(NamedTuple):
198
+ hits: int
199
+ total: int
200
+
201
+ @property
202
+ def hit_ratio(self) -> float:
203
+ if self.total == 0:
204
+ return 0
205
+
206
+ return self.hits / self.total
207
+
208
+
209
class LRUCache(Generic[_K, _V]):
210
+ """Note: This class is not thread safe!"""
211
212
def __init__(self, capacity: int) -> None:
213
self.cache = OrderedDict[_K, _V]()
214
self.pinned_items = set[_K]()
215
self.capacity = capacity
216
217
+ self._hits = 0
218
+ self._total = 0
219
+
220
def __contains__(self, key: _K) -> bool:
221
return key in self.cache
222
@@ -218,6 +234,9 @@ class LRUCache(Generic[_K, _V]):
234
def __delitem__(self, key: _K) -> None:
235
self.pop(key)
236
237
+ def stat(self) -> CacheInfo:
238
+ return CacheInfo(hits=self._hits, total=self._total)
239
+
240
def touch(self, key: _K) -> None:
241
self.cache.move_to_end(key)
242
@@ -226,8 +245,12 @@ class LRUCache(Generic[_K, _V]):
245
if key in self.cache:
246
value = self.cache[key]
247
self.cache.move_to_end(key)
248
+
249
+ self._hits += 1
250
else:
251
value = default
252
+
253
+ self._total += 1
254
return value
255
256
def put(self, key: _K, value: _V) -> None: