@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 101418096f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Support caching in merged multi-modal processor (#11396)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 28, 2024 at 01:22 UTC 101418096ffe3c83b6d541e1303b10e9d5e03861
20 files changed +1455 -448
docs/source/conf.py
+2 -1
@@ -191,6 +191,7 @@ def linkcode_resolve(domain, info):
191
192 # Mock out external dependencies here, otherwise the autodoc pages may be blank.
193 autodoc_mock_imports = [
194 + "blake3",
195 "compressed_tensors",
196 "cpuinfo",
197 "cv2",
@@ -207,7 +208,7 @@ autodoc_mock_imports = [
208 "tensorizer",
209 "pynvml",
210 "outlines",
210 - "xgrammar,"
211 + "xgrammar",
212 "librosa",
213 "soundfile",
214 "gguf",
docs/source/design/multimodal/multimodal_index.md
+12 -12
@@ -45,39 +45,39 @@ adding_multimodal_plugin
45 ### Base Classes
46
47 ```{eval-rst}
48 -.. autodata:: vllm.multimodal.NestedTensors
48 +.. automodule:: vllm.multimodal.base
49 + :members:
50 + :show-inheritance:
51 ```
52
51 -```{eval-rst}
52 -.. autodata:: vllm.multimodal.BatchedTensorInputs
53 -```
53 +### Input Classes
54
55 ```{eval-rst}
56 -.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
56 +.. automodule:: vllm.multimodal.inputs
57 :members:
58 :show-inheritance:
59 ```
60
61 -```{eval-rst}
62 -.. autodata:: vllm.multimodal.MultiModalDataDict
63 -```
61 +### Audio Classes
62
63 ```{eval-rst}
66 -.. autoclass:: vllm.multimodal.MultiModalKwargs
64 +.. automodule:: vllm.multimodal.audio
65 :members:
66 :show-inheritance:
67 ```
68
69 +### Image Classes
70 +
71 ```{eval-rst}
72 -.. autoclass:: vllm.multimodal.MultiModalPlugin
72 +.. automodule:: vllm.multimodal.image
73 :members:
74 :show-inheritance:
75 ```
76
77 -### Image Classes
77 +### Video Classes
78
79 ```{eval-rst}
80 -.. automodule:: vllm.multimodal.image
80 +.. automodule:: vllm.multimodal.video
81 :members:
82 :show-inheritance:
83 ```
docs/source/models/supported_models.md
+1 -2
@@ -755,8 +755,7 @@ vLLM currently only supports adding LoRA to the language backbone of multimodal
755 ```
756
757 ```{note}
758 -To use {code}`TIGER-Lab/Mantis-8B-siglip-llama3`, you have to install their GitHub repo ({code}`pip install git+https://github.com/TIGER-AI-Lab/Mantis.git`)
759 -and pass {code}`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
758 +To use {code}`TIGER-Lab/Mantis-8B-siglip-llama3`, you have pass {code}`--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
759 ```
760
761 ```{note}
tests/entrypoints/openai/test_vision_embedding.py
+2 -2
@@ -91,5 +91,5 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
91 assert len(embeddings.data) == 1
92 assert len(embeddings.data[0].embedding) == 3072
93 assert embeddings.usage.completion_tokens == 0
94 - assert embeddings.usage.prompt_tokens == 765
95 - assert embeddings.usage.total_tokens == 765
94 + assert embeddings.usage.prompt_tokens == 764
95 + assert embeddings.usage.total_tokens == 764
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py
+1 -1
@@ -30,7 +30,7 @@ def get_max_qwen2_vl_image_tokens():
30
31
32 @pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
33 - ({}, 1225),
33 + ({}, 16384),
34 ({
35 MIN_PIXELS: 64**2,
36 MAX_PIXELS: 512**2
tests/models/decoder_only/vision_language/test_models.py
+3 -1
@@ -201,6 +201,7 @@ VLM_TEST_SETTINGS = {
201 vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
202 num_logprobs=10,
203 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
204 + marks=[large_gpu_mark(min_gb=48)],
205 ),
206 "glm4": VLMTestInfo(
207 models=["THUDM/glm-4v-9b"],
@@ -212,7 +213,7 @@ VLM_TEST_SETTINGS = {
213 dtype="bfloat16",
214 get_stop_token_ids=lambda tok: [151329, 151336, 151338],
215 patch_hf_runner=model_utils.glm_patch_hf_runner,
215 - marks=[large_gpu_mark(min_gb=48)],
216 + marks=[large_gpu_mark(min_gb=32)],
217 ),
218 "h2ovl": VLMTestInfo(
219 models = [
@@ -261,6 +262,7 @@ VLM_TEST_SETTINGS = {
262 dtype="bfloat16",
263 use_tokenizer_eos=True,
264 patch_hf_runner=model_utils.internvl_patch_hf_runner,
265 + marks=[large_gpu_mark(min_gb=32)],
266 ),
267 "llava_next": VLMTestInfo(
268 models=["llava-hf/llava-v1.6-mistral-7b-hf"],
tests/multimodal/test_processing.py
+203 -6
@@ -1,12 +1,20 @@
1 +from functools import partial
2 from typing import cast
3
4 +import numpy as np
5 import pytest
4 -
5 -from vllm.multimodal.processing import (PromptReplacement, _PlaceholderInfo,
6 - find_text_matches, find_token_matches,
7 - iter_placeholders, iter_token_matches,
6 +from PIL import Image
7 +
8 +from vllm.config import ModelConfig
9 +from vllm.inputs import InputProcessingContext
10 +from vllm.multimodal import MULTIMODAL_REGISTRY
11 +from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
12 + _PlaceholderInfo, find_text_matches,
13 + find_token_matches, iter_placeholders,
14 + iter_token_matches,
15 replace_text_matches,
16 replace_token_matches)
17 +from vllm.multimodal.utils import cached_get_tokenizer
18 from vllm.transformers_utils.tokenizer import AnyTokenizer
19 from vllm.utils import full_groupby
20
@@ -457,6 +465,7 @@ def test_find_replace_tokens(
465 ),
466 ]
467 )
468 +# yapf: enable
469 def test_iter_placeholders(
470 repl_by_key,
471 prompt,
@@ -475,11 +484,199 @@ def test_iter_placeholders(
484 prompt_repls,
485 prompt,
486 # Effectively match all occurrences in the prompt
478 - {key: 3 for key in repl_by_key},
479 - ))
487 + {key: 3
488 + for key in repl_by_key},
489 + ))
490
491 # Only displayed on error
492 print("result:", result)
493
494 # Manually constructed results
495 assert result == expected
496 +
497 +
498 +def _rand_img(rng: np.random.RandomState, min_wh: int, max_wh: int):
499 + w, h = rng.randint(min_wh, max_wh, size=(2, ))
500 + arr = rng.randint(0, 255, size=(w, h, 3), dtype=np.uint8)
501 + return Image.fromarray(arr)
502 +
503 +
504 +def _rand_video(
505 + rng: np.random.RandomState,
506 + min_frames: int,
507 + max_frames: int,
508 + min_wh: int,
509 + max_wh: int,
510 +):
511 + # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
512 + num_frames = rng.randint(min_frames, max_frames)
513 + num_frames = (num_frames // 2) * 2
514 +
515 + w, h = rng.randint(min_wh, max_wh, size=(2, ))
516 + return rng.randint(0, 255, size=(num_frames, w, h, 3), dtype=np.uint8)
517 +
518 +
519 +def _rand_audio(
520 + rng: np.random.RandomState,
521 + min_len: int,
522 + max_len: int,
523 + sr: int,
524 +):
525 + audio_len = rng.randint(min_len, max_len)
526 + return rng.rand(audio_len), sr
527 +
528 +
529 +def _test_processing_cache_correctness(
530 + model_id: str,
531 + modalities: set[str],
532 + hit_rate: float,
533 + num_batches: int,
534 + simplify_rate: float,
535 +):
536 + if model_id == "TIGER-Lab/Mantis-8B-siglip-llama3":
537 + hf_overrides = {"architectures": ["MantisForConditionalGeneration"]}
538 + else:
539 + hf_overrides = {}
540 +
541 + model_config = ModelConfig(
542 + model_id,
543 + task="auto",
544 + tokenizer=model_id,
545 + tokenizer_mode="auto",
546 + trust_remote_code=True,
547 + seed=0,
548 + dtype="float16",
549 + revision=None,
550 + hf_overrides=hf_overrides,
551 + )
552 + model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
553 +
554 + processor_factory = MULTIMODAL_REGISTRY._processor_factories[model_cls]
555 + ctx = InputProcessingContext(
556 + model_config,
557 + tokenizer=cached_get_tokenizer(model_config.tokenizer),
558 + )
559 + # Ensure that it can fit all of the data
560 + cache = ProcessingCache(capacity=1 << 30)
561 +
562 + baseline_processor = processor_factory(ctx, cache=None)
563 + cached_processor = processor_factory(ctx, cache=cache)
564 +
565 + rng = np.random.RandomState(0)
566 +
567 + input_to_hit = {
568 + "image": Image.new("RGB", size=(128, 128)),
569 + "video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
570 + "audio": (np.zeros((512, )), 16000),
571 + }
572 + input_factory = {
573 + "image":
574 + partial(_rand_img, rng, min_wh=128, max_wh=256),
575 + "video":
576 + partial(_rand_video,
577 + rng,
578 + min_frames=2,
579 + max_frames=8,
580 + min_wh=128,
581 + max_wh=256),
582 + "audio":
583 + partial(_rand_audio, rng, min_len=256, max_len=512, sr=16000),
584 + }
585 + input_max_count = {
586 + "image": 3,
587 + "video": 3,
588 + "audio": 3,
589 + }
590 +
591 + for batch_idx in range(num_batches):
592 + mm_data = {
593 + k:
594 + [(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
595 + for _ in range(rng.randint(input_max_count[k]))]
596 + for k in modalities
597 + }
598 +
599 + mm_counts = {k: len(vs) for k, vs in mm_data.items()}
600 + prompt = baseline_processor._get_dummy_mm_inputs(mm_counts).prompt_text
601 +
602 + # Drop unnecessary keys and test single -> multi conversion
603 + if rng.rand() < simplify_rate:
604 + for k in list(mm_data.keys()):
605 + if not mm_data[k]:
606 + del mm_data[k]
607 + elif len(mm_data[k]) == 1:
608 + mm_data[k] = mm_data[k][0]
609 +
610 + baseline_result = baseline_processor.apply(
611 + prompt,
612 + mm_data=mm_data,
613 + hf_processor_mm_kwargs={},
614 + )
615 + cached_result = cached_processor.apply(
616 + prompt,
617 + mm_data=mm_data,
618 + hf_processor_mm_kwargs={},
619 + )
620 +
621 + assert baseline_result == cached_result, (
622 + f"Failed ({batch_idx=}, {mm_data=})")
623 +
624 +
625 +# yapf: disable
626 +@pytest.mark.parametrize(("model_id", "modalities"), [
627 + ("llava-hf/llava-1.5-7b-hf", {"image"}),
628 + ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image"}),
629 + ("mistral-community/pixtral-12b", {"image"}),
630 + ("Qwen/Qwen2-VL-2B-Instruct", {"image", "video"}),
631 + ("Qwen/Qwen2-Audio-7B-Instruct", {"audio"}),
632 + ("fixie-ai/ultravox-v0_3", {"audio"}),
633 +])
634 +@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
635 +@pytest.mark.parametrize("num_batches", [32])
636 +@pytest.mark.parametrize("simplify_rate", [1.0])
637 +# yapf: enable
638 +def test_processing_cache_correctness(
639 + model_id: str,
640 + modalities: set[str],
641 + hit_rate: float,
642 + num_batches: int,
643 + simplify_rate: float,
644 +):
645 + _test_processing_cache_correctness(
646 + model_id,
647 + modalities,
648 + hit_rate=hit_rate,
649 + num_batches=num_batches,
650 + simplify_rate=simplify_rate,
651 + )
652 +
653 +
654 +# yapf: disable
655 +@pytest.mark.parametrize(("model_id", "modalities"), [
656 + ("microsoft/Phi-3-vision-128k-instruct", {"image"}),
657 +])
658 +@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
659 +@pytest.mark.parametrize("num_batches", [32])
660 +@pytest.mark.parametrize("simplify_rate", [1.0])
661 +# yapf: enable
662 +def test_processing_cache_correctness_phi3v(
663 + model_id: str,
664 + modalities: set[str],
665 + hit_rate: float,
666 + num_batches: int,
667 + simplify_rate: float,
668 +):
669 + # HACK - this is an attempted workaround for the following bug
670 + # https://github.com/huggingface/transformers/issues/34307
671 + from transformers import AutoImageProcessor # noqa: F401
672 + from transformers import AutoProcessor # noqa: F401
673 +
674 + AutoImageProcessor.from_pretrained(model_id, trust_remote_code=True)
675 +
676 + _test_processing_cache_correctness(
677 + model_id,
678 + modalities,
679 + hit_rate=hit_rate,
680 + num_batches=num_batches,
681 + simplify_rate=simplify_rate,
682 + )
vllm/inputs/registry.py
+11 -11
@@ -99,6 +99,9 @@ class InputContext:
99
100 merged_kwargs = {**base_kwargs, **kwargs}
101
102 + if isinstance(typ, type):
103 + merged_kwargs["processor_cls"] = typ
104 +
105 hf_processor = cached_get_processor(
106 self.model_config.model,
107 trust_remote_code=self.model_config.trust_remote_code,
@@ -132,10 +135,13 @@ class InputProcessingContext(InputContext):
135 def call_hf_processor(
136 self,
137 hf_processor: ProcessorMixin,
135 - prompt: str,
136 - processor_data: Mapping[str, object],
137 - inference_kwargs: Mapping[str, object],
138 + data: Mapping[str, object],
139 + kwargs: Mapping[str, object] = {},
140 ) -> BatchFeature:
141 + """
142 + Call :code:`hf_processor` on the prompt :code:`data`
143 + (text, image, audio...) with configurable options :code:`kwargs`.
144 + """
145 assert callable(hf_processor)
146
147 base_kwargs = self.model_config.mm_processor_kwargs
@@ -144,21 +150,15 @@ class InputProcessingContext(InputContext):
150
151 merged_kwargs = resolve_mm_processor_kwargs(
152 base_kwargs,
147 - inference_kwargs,
153 + kwargs,
154 hf_processor,
155 requires_kw_only=False,
156 allow_var_kwargs=True,
157 )
158
159 try:
154 - return hf_processor(
155 - text=prompt,
156 - **processor_data,
157 - **merged_kwargs,
158 - return_tensors="pt",
159 - )
160 + return hf_processor(**data, **merged_kwargs, return_tensors="pt")
161 except Exception as exc:
161 - data = dict(text=prompt, **processor_data)
162 msg = (f"Failed to apply {type(hf_processor).__name__} "
163 f"on data={data} with kwargs={merged_kwargs}")
164
vllm/model_executor/models/llava.py
+110 -68
@@ -1,5 +1,4 @@
1 from functools import cached_property
2 -from types import MethodType
2 from typing import (Iterable, List, Literal, Mapping, Optional, Protocol, Set,
3 Tuple, TypedDict, Union)
4
@@ -7,7 +6,7 @@ import torch
6 import torch.nn as nn
7 from transformers import (BatchFeature, CLIPVisionConfig, LlavaConfig,
8 PixtralVisionConfig, PretrainedConfig,
10 - ProcessorMixin, SiglipVisionConfig)
9 + SiglipVisionConfig)
10 from transformers.models.llava import LlavaProcessor
11 from transformers.models.pixtral import PixtralProcessor
12
@@ -21,10 +20,12 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
21 from vllm.model_executor.sampling_metadata import SamplingMetadata
22 from vllm.multimodal import MULTIMODAL_REGISTRY
24 -from vllm.multimodal.inputs import NestedTensors
23 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
24 + MultiModalFieldConfig, MultiModalInputsV2,
25 + MultiModalKwargs, NestedTensors)
26 from vllm.multimodal.processing import (BaseMultiModalProcessor,
26 - MultiModalDataItems, ProcessorInputs,
27 - PromptReplacement)
27 + ProcessorInputs, PromptReplacement,
28 + full_groupby_modality)
29 from vllm.sequence import IntermediateTensors
30
31 from .clip import (CLIPVisionModel, dummy_image_for_clip,
@@ -116,36 +117,54 @@ def get_max_llava_image_tokens(ctx: InputContext):
117
118 class LlavaMultiModalProcessor(BaseMultiModalProcessor):
119
119 - def _patch_pixtral_processor(self, hf_processor: PixtralProcessor):
120 - if getattr(hf_processor, "__is_patched__", False):
121 - return # Already patched
122 -
123 - image_processor = hf_processor.image_processor # type: ignore
124 - orig_preprocess = image_processor.preprocess
120 + def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
121 + return self.ctx.get_hf_processor((LlavaProcessor, PixtralProcessor))
122
126 - def preprocess(__self, *args, **kwargs):
127 - hf_inputs = orig_preprocess(*args, **kwargs)
128 - hf_inputs["is_pixtral"] = torch.tensor(True)
129 - return hf_inputs
123 + def _call_hf_processor(
124 + self,
125 + prompt: str,
126 + mm_data: Mapping[str, object],
127 + mm_kwargs: Mapping[str, object],
128 + ) -> BatchFeature:
129 + processed_outputs = super()._call_hf_processor(
130 + prompt=prompt,
131 + mm_data=mm_data,
132 + mm_kwargs=mm_kwargs,
133 + )
134
131 - image_processor.preprocess = MethodType(preprocess, image_processor)
135 + # NOTE: pixel_values=None for MLlavaProcessor
136 + pixel_values = processed_outputs.get("pixel_values")
137 + if pixel_values is not None:
138 + images = mm_data["images"]
139 + assert isinstance(images, list)
140
133 - hf_processor.__is_patched__ = True # type: ignore
141 + if isinstance(self._get_hf_processor(), PixtralProcessor):
142 + # Original output: (1, num_images, C, H, W)
143 + # New output: (num_images, C, H, W)
144 + assert (isinstance(pixel_values, list)
145 + and len(pixel_values) == 1
146 + and isinstance(pixel_values[0], list)
147 + and len(pixel_values[0]) == len(images))
148
135 - def _get_hf_processor(self) -> Union[LlavaProcessor, PixtralProcessor]:
136 - hf_processor = self.ctx.get_hf_processor(
137 - (LlavaProcessor, PixtralProcessor))
149 + processed_outputs["pixel_values"] = pixel_values[0]
150
139 - if isinstance(hf_processor, PixtralProcessor):
140 - self._patch_pixtral_processor(hf_processor)
151 + return processed_outputs
152
142 - return hf_processor
153 + def _get_mm_fields_config(
154 + self,
155 + hf_inputs: BatchFeature,
156 + hf_processor_mm_kwargs: Mapping[str, object],
157 + ) -> Mapping[str, MultiModalFieldConfig]:
158 + return dict(
159 + pixel_values=MultiModalFieldConfig.batched("image"),
160 + image_embeds=MultiModalFieldConfig.batched("image"),
161 + )
162
163 def _get_prompt_replacements(
164 self,
165 mm_items: MultiModalDataItems,
147 - hf_inputs: BatchFeature,
148 - mm_processor_kwargs: Mapping[str, object],
166 + hf_processor_mm_kwargs: Mapping[str, object],
167 + out_mm_kwargs: MultiModalKwargs,
168 ) -> list[PromptReplacement]:
169 hf_config = self.ctx.get_hf_config(LlavaConfig)
170 image_token_id = hf_config.image_token_index
@@ -200,7 +219,7 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
219 ) -> ProcessorInputs:
220 hf_config = self.ctx.get_hf_config(LlavaConfig)
221 vision_config = hf_config.vision_config
203 - num_images = mm_counts["image"]
222 + num_images = mm_counts.get("image", 0)
223
224 if isinstance(vision_config, CLIPVisionConfig):
225 data = dummy_image_for_clip(vision_config, num_images)
@@ -218,7 +237,6 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
237 return ProcessorInputs(
238 prompt_text=image_token * num_images,
239 mm_data=data,
221 - mm_processor_kwargs={},
240 )
241
242
@@ -379,7 +397,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
397 def _parse_and_validate_image_input(
398 self, **kwargs: object) -> Optional[LlavaImageInputs]:
399 pixel_values = kwargs.pop("pixel_values", None)
382 - is_pixtral = kwargs.pop("is_pixtral", torch.tensor([False]))
400 image_embeds = kwargs.pop("image_embeds", None)
401
402 if pixel_values is None and image_embeds is None:
@@ -390,33 +407,6 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
407 raise ValueError("Incorrect type of pixel values. "
408 f"Got type: {type(pixel_values)}")
409
393 - assert isinstance(is_pixtral, torch.Tensor)
394 - if is_pixtral.any():
395 - images = pixel_values
396 -
397 - def flatten_to_3d_tensors(item):
398 - if isinstance(item, torch.Tensor):
399 - if item.dim() >= 3:
400 - return [t for t in item.view(-1, *item.shape[-3:])]
401 - else:
402 - raise ValueError(
403 - f"Unexpected tensor dimension: {item.dim()}")
404 - elif isinstance(item, list):
405 - return [
406 - t for subitem in item
407 - for t in flatten_to_3d_tensors(subitem)
408 - ]
409 - else:
410 - raise ValueError(f"Unexpected type: {type(item)}")
411 -
412 - # Restructure the batched images into a list of lists of images
413 - images = flatten_to_3d_tensors(pixel_values)
414 -
415 - return LlavaImagePixelInputs(
416 - type="pixel_values",
417 - data=images,
418 - )
419 -
410 return LlavaImagePixelInputs(
411 type="pixel_values",
412 data=self._validate_pixel_values(
@@ -586,19 +576,71 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
576
577 class MantisMultiModalProcessor(LlavaMultiModalProcessor):
578
589 - def _get_hf_processor(self) -> ProcessorMixin:
590 - try:
591 - from mantis.models.mllava import MLlavaProcessor
592 - except ModuleNotFoundError as exc:
593 - raise ModuleNotFoundError(
594 - "You need to `pip install "
595 - "git+https://github.com/TIGER-AI-Lab/Mantis.git` "
596 - "to use this model") from exc
597 -
598 - processor = MLlavaProcessor.from_pretrained(
599 - self.ctx.model_config.tokenizer)
600 - assert isinstance(processor, ProcessorMixin)
601 - return processor
579 + def _get_hf_processor(self):
580 + return self.ctx.get_hf_processor(LlavaProcessor)
581 +
582 + def apply(
583 + self,
584 + prompt_text: str,
585 + mm_data: MultiModalDataDict,
586 + hf_processor_mm_kwargs: Mapping[str, object],
587 + ) -> MultiModalInputsV2:
588 + hf_config = self.ctx.get_hf_config(LlavaConfig)
589 + image_token_id = hf_config.image_token_index
590 + max_image_tokens = get_max_llava_image_tokens(self.ctx)
591 +
592 + result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
593 +
594 + mm_items = self._get_mm_items(mm_data)
595 + mm_item_counts = mm_items.get_item_counts()
596 + mm_kwargs = result["mm_kwargs"]
597 +
598 + # We reimplement the functionality of MLlavaProcessor from
599 + # https://github.com/TIGER-AI-Lab/Mantis.git
600 + def get_replacement_mantis(item_idx: int):
601 + return "".join([
602 + f"(image {item_idx+1}: <Image>", # 7 tokens
603 + "<image>" * max_image_tokens,
604 + "</Image>)", # 3 tokens
605 + ])
606 +
607 + mantis_repls = self._bind_prompt_replacements([
608 + PromptReplacement(
609 + modality="image",
610 + target=[image_token_id] * max_image_tokens,
611 + replacement=get_replacement_mantis,
612 + )
613 + ])
614 +
615 + prompt_ids, prompt_text, _ = self._apply_prompt_replacements(
616 + result["prompt_token_ids"],
617 + mantis_repls,
618 + mm_item_counts,
619 + )
620 +
621 + unbound_orig_repls = self._get_prompt_replacements(
622 + mm_items,
623 + hf_processor_mm_kwargs,
624 + mm_kwargs,
625 + )
626 + orig_repls = self._bind_prompt_replacements(unbound_orig_repls)
627 +
628 + all_placeholders = self._find_placeholders(orig_repls, prompt_ids,
629 + mm_item_counts)
630 + assert len(all_placeholders) == mm_item_counts.get("image", 0)
631 +
632 + mm_placeholders = {
633 + modality: [item.to_range() for item in items]
634 + for modality, items in full_groupby_modality(all_placeholders)
635 + }
636 +
637 + return MultiModalInputsV2(
638 + type="multimodal",
639 + prompt=prompt_text,
640 + prompt_token_ids=prompt_ids,
641 + mm_kwargs=mm_kwargs,
642 + mm_placeholders=mm_placeholders,
643 + )
644
645
646 # To use this model, please use
vllm/model_executor/models/phi3v.py
+82 -25
@@ -12,9 +12,9 @@
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 +from collections.abc import Iterable, Mapping, Sequence
16 from functools import cached_property
16 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
17 - TypedDict, Union)
17 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
18
19 import torch
20 import torch.nn as nn
@@ -32,10 +32,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32 from vllm.model_executor.models.clip import CLIPVisionModel
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.inputs import NestedTensors
35 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
36 + MultiModalFieldConfig, MultiModalInputsV2,
37 + MultiModalKwargs, NestedTensors,
38 + PlaceholderRange)
39 from vllm.multimodal.processing import (BaseMultiModalProcessor,
37 - MultiModalDataItems, ProcessorInputs,
38 - PromptReplacement)
40 + ProcessorInputs, PromptReplacement,
41 + _BoundPromptReplacement,
42 + _PlaceholderInfo)
43 from vllm.sequence import IntermediateTensors
44 from vllm.utils import is_list_of
45
@@ -306,11 +310,11 @@ def get_max_phi3v_image_tokens(
310 *,
311 num_crops: Optional[int] = None,
312 ) -> int:
309 - mm_processor_kwargs = {}
313 + hf_processor_mm_kwargs = {}
314 if num_crops:
311 - mm_processor_kwargs["num_crops"] = num_crops
315 + hf_processor_mm_kwargs["num_crops"] = num_crops
316
313 - processor = ctx.get_hf_processor(**mm_processor_kwargs)
317 + processor = ctx.get_hf_processor(**hf_processor_mm_kwargs)
318
319 return processor.calc_num_image_tokens_from_image_size(
320 width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
@@ -331,39 +335,50 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
335
336 def _call_hf_processor(
337 self,
334 - hf_processor: ProcessorMixin,
338 prompt: str,
336 - processor_data: Mapping[str, object],
337 - mm_processor_kwargs: Mapping[str, object],
339 + mm_data: Mapping[str, object],
340 + mm_kwargs: Mapping[str, object],
341 ) -> BatchFeature:
342 processed_outputs = super()._call_hf_processor(
340 - hf_processor,
343 prompt=prompt,
342 - processor_data=processor_data,
343 - mm_processor_kwargs=mm_processor_kwargs,
344 + mm_data=mm_data,
345 + mm_kwargs=mm_kwargs,
346 )
347
348 + input_ids = processed_outputs["input_ids"]
349 + assert isinstance(input_ids, torch.Tensor)
350 +
351 # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
352 # which will cause OverflowError when decoding the prompt_ids.
353 # Therefore, we need to do an early replacement here
349 - token_ids = processed_outputs['input_ids']
350 - token_ids[token_ids < 0] = _IMAGE_TOKEN_ID
351 - processed_outputs['input_ids'] = token_ids
354 + input_ids.masked_fill_(input_ids < 0, _IMAGE_TOKEN_ID)
355
356 return processed_outputs
357
358 + def _get_mm_fields_config(
359 + self,
360 + hf_inputs: BatchFeature,
361 + hf_processor_mm_kwargs: Mapping[str, object],
362 + ) -> Mapping[str, MultiModalFieldConfig]:
363 + return dict(
364 + pixel_values=MultiModalFieldConfig.batched("image"),
365 + image_sizes=MultiModalFieldConfig.batched("image"),
366 + image_embeds=MultiModalFieldConfig.batched("image"),
367 + )
368 +
369 def _get_prompt_replacements(
370 self,
371 mm_items: MultiModalDataItems,
358 - hf_inputs: BatchFeature,
359 - mm_processor_kwargs: Mapping[str, object],
372 + hf_processor_mm_kwargs: Mapping[str, object],
373 + out_mm_kwargs: MultiModalKwargs,
374 ) -> list[PromptReplacement]:
375 hf_processor = self._get_hf_processor()
376 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
377 image_processor = hf_processor.image_processor # type: ignore
378
365 - mm_config = self.ctx.get_mm_config()
366 - max_images = mm_config.limit_per_prompt.get("image", 1)
379 + tokenizer = self._get_tokenizer()
380 + bos_token_id = tokenizer.bos_token_id
381 + assert isinstance(bos_token_id, int)
382
383 def get_replacement_phi3v(item_idx: int):
384 image_size = mm_items.get_image_size(item_idx)
@@ -372,21 +387,44 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
387 height=image_size.height,
388 )
389
375 - return [_IMAGE_TOKEN_ID] * num_tokens
390 + return [_IMAGE_TOKEN_ID] * num_tokens + [bos_token_id]
391
392 return [
393 PromptReplacement(
394 modality="image",
395 target=image_token,
396 replacement=get_replacement_phi3v,
382 - ) for image_token in image_tokens[:max_images]
397 + ) for image_token in image_tokens[:len(mm_items.images)]
398 ]
399
400 + def _apply_prompt_replacements(
401 + self,
402 + token_ids: list[int],
403 + prompt_repls: Sequence[_BoundPromptReplacement],
404 + mm_item_counts: Mapping[str, int],
405 + ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
406 + token_ids, text, placeholders = super()._apply_prompt_replacements(
407 + token_ids=token_ids,
408 + prompt_repls=prompt_repls,
409 + mm_item_counts=mm_item_counts,
410 + )
411 +
412 + # Keep the behavior in line with HF processor
413 + if text.startswith("<s> <|image|>"):
414 + text = text.replace("<s> <|image|>", "<s><|image|>", 1)
415 + token_ids = [token_ids[0], *token_ids[2:]]
416 + placeholders = [
417 + _PlaceholderInfo(p.modality, p.start_idx - 1, p.replacement)
418 + for p in placeholders
419 + ]
420 +
421 + return token_ids, text, placeholders
422 +
423 def _get_dummy_mm_inputs(
424 self,
425 mm_counts: Mapping[str, int],
426 ) -> ProcessorInputs:
389 - num_images = mm_counts["image"]
427 + num_images = mm_counts.get("image", 0)
428
429 data = dummy_image_for_clip(
430 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
@@ -401,9 +439,28 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
439 return ProcessorInputs(
440 prompt_text="".join(image_tokens[:num_images]),
441 mm_data=data,
404 - mm_processor_kwargs={},
442 )
443
444 + def apply(
445 + self,
446 + prompt_text: str,
447 + mm_data: MultiModalDataDict,
448 + hf_processor_mm_kwargs: Mapping[str, object],
449 + ) -> MultiModalInputsV2:
450 + result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
451 +
452 + # Only <|image|> tokens should be considered as placeholders,
453 + # so we ignore the trailing bos_token_id
454 + result["mm_placeholders"] = {
455 + modality: [
456 + PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
457 + for p in ps
458 + ]
459 + for modality, ps in result["mm_placeholders"].items()
460 + }
461 +
462 + return result
463 +
464
465 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
466 @MULTIMODAL_REGISTRY.register_processor(Phi3VMultiModalProcessor)
vllm/model_executor/models/qwen.py
+2 -2
@@ -225,7 +225,7 @@ class VisualAttentionBlock(nn.Module):
225 d_model: int,
226 n_head: int,
227 mlp_ratio: float = 4.0,
228 - norm_layer: Callable = nn.LayerNorm,
228 + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
229 quant_config: Optional[QuantizationConfig] = None,
230 ):
231 super().__init__()
@@ -266,7 +266,7 @@ class TransformerBlock(nn.Module):
266 layers: int,
267 heads: int,
268 mlp_ratio: float = 4.0,
269 - norm_layer: Callable = nn.LayerNorm,
269 + norm_layer: Callable[[int], nn.Module] = nn.LayerNorm,
270 quant_config: Optional[QuantizationConfig] = None,
271 ):
272 super().__init__()
vllm/model_executor/models/qwen2_audio.py
+40 -25
@@ -26,7 +26,7 @@ from typing import (Any, Iterable, List, Mapping, Optional, Set, Tuple,
26 import numpy as np
27 import torch
28 import torch.nn as nn
29 -from transformers import BatchFeature, ProcessorMixin
29 +from transformers import BatchFeature
30 from transformers.models.qwen2_audio import (Qwen2AudioConfig,
31 Qwen2AudioEncoder,
32 Qwen2AudioProcessor)
@@ -38,10 +38,10 @@ from vllm.inputs import InputContext
38 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.multimodal import MULTIMODAL_REGISTRY
41 -from vllm.multimodal.inputs import NestedTensors
41 +from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
42 + MultiModalKwargs, NestedTensors)
43 from vllm.multimodal.processing import (BaseMultiModalProcessor,
43 - MultiModalDataItems, ProcessorInputs,
44 - PromptReplacement)
44 + ProcessorInputs, PromptReplacement)
45 from vllm.sequence import IntermediateTensors
46
47 from .interfaces import SupportsMultiModal, SupportsPP
@@ -73,7 +73,7 @@ class Qwen2AudioMultiModalProjector(nn.Module):
73
74
75 # From Qwen2AudioEncoder._get_feat_extract_output_lengths
76 -def _get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
76 +def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
77 feat_lengths = (input_lengths - 1) // 2 + 1
78 output_lengths = (feat_lengths - 2) // 2 + 1
79 return feat_lengths, output_lengths
@@ -88,13 +88,18 @@ def get_max_qwen2_audio_audio_tokens(ctx: InputContext) -> int:
88
89 class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
90
91 - def _get_hf_processor(self) -> Qwen2AudioProcessor:
91 + def _get_hf_processor(
92 + self,
93 + *,
94 + # Ignored in initialization
95 + sampling_rate: Optional[int] = None,
96 + ) -> Qwen2AudioProcessor:
97 return self.ctx.get_hf_processor(Qwen2AudioProcessor)
98
99 def _get_feature_extractor(self) -> WhisperFeatureExtractor:
100 return self._get_hf_processor().feature_extractor # type: ignore
101
97 - def _get_processor_data(
102 + def _get_hf_mm_data(
103 self,
104 mm_items: MultiModalDataItems,
105 ) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -102,50 +107,61 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
107 feature_extractor = self._get_feature_extractor()
108 mm_items.resample_audios(feature_extractor.sampling_rate)
109
105 - return super()._get_processor_data(mm_items)
110 + return super()._get_hf_mm_data(mm_items)
111
112 def _call_hf_processor(
113 self,
109 - hf_processor: ProcessorMixin,
114 prompt: str,
111 - processor_data: Mapping[str, object],
112 - mm_processor_kwargs: Mapping[str, object],
115 + mm_data: Mapping[str, object],
116 + mm_kwargs: Mapping[str, object],
117 ) -> BatchFeature:
114 - processor_data = dict(processor_data)
115 - audios = processor_data.pop("audios", [])
118 + mm_data = dict(mm_data)
119 + audios = mm_data.pop("audios", [])
120
121 if audios:
118 - processor_data["audios"] = audios
122 + mm_data["audios"] = audios
123
124 feature_extractor = self._get_feature_extractor()
121 - mm_processor_kwargs = dict(
122 - **mm_processor_kwargs,
125 + mm_kwargs = dict(
126 + **mm_kwargs,
127 sampling_rate=feature_extractor.sampling_rate,
128 )
129 else:
130 # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
131 pass
132
129 - return super()._call_hf_processor(
130 - hf_processor,
133 + processed_outputs = super()._call_hf_processor(
134 prompt=prompt,
132 - processor_data=processor_data,
133 - mm_processor_kwargs=mm_processor_kwargs,
135 + mm_data=mm_data,
136 + mm_kwargs=mm_kwargs,
137 + )
138 +
139 + return processed_outputs
140 +
141 + def _get_mm_fields_config(
142 + self,
143 + hf_inputs: BatchFeature,
144 + hf_processor_mm_kwargs: Mapping[str, object],
145 + ) -> Mapping[str, MultiModalFieldConfig]:
146 + return dict(
147 + input_features=MultiModalFieldConfig.batched("audio"),
148 + feature_attention_mask=MultiModalFieldConfig.batched("audio"),
149 )
150
151 def _get_prompt_replacements(
152 self,
153 mm_items: MultiModalDataItems,
139 - hf_inputs: BatchFeature,
140 - mm_processor_kwargs: Mapping[str, object],
154 + hf_processor_mm_kwargs: Mapping[str, object],
155 + out_mm_kwargs: MultiModalKwargs,
156 ) -> list[PromptReplacement]:
157 hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
158 placeholder = hf_config.audio_token_index
159
145 - feature_attention_mask = hf_inputs.get("feature_attention_mask")
160 + feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
161 if feature_attention_mask is None:
162 audio_output_lengths = []
163 else:
164 + assert isinstance(feature_attention_mask, torch.Tensor)
165 _, audio_output_lengths = _get_feat_extract_output_lengths(
166 feature_attention_mask.sum(-1))
167
@@ -168,14 +184,13 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
184 sampling_rate = feature_extractor.sampling_rate
185 audio_len = feature_extractor.chunk_length * sampling_rate
186
171 - audio_count = mm_counts["audio"]
187 + audio_count = mm_counts.get("audio", 0)
188 audio = np.zeros(audio_len)
189 data = {"audio": [audio] * audio_count}
190
191 return ProcessorInputs(
192 prompt_text="<|AUDIO|>" * audio_count,
193 mm_data=data,
178 - mm_processor_kwargs={},
194 )
195
196
vllm/model_executor/models/qwen2_vl.py
+78 -37
@@ -22,9 +22,10 @@
22 # limitations under the License.
23 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
24 from functools import cached_property, partial
25 -from typing import (Any, Iterable, List, Literal, Mapping, Optional, Set,
26 - Tuple, Type, TypedDict, Union)
25 +from typing import (Any, Callable, Iterable, List, Literal, Mapping, Optional,
26 + Set, Tuple, Type, TypedDict, Union)
27
28 +import numpy as np
29 import torch
30 import torch.nn as nn
31 import torch.nn.functional as F
@@ -54,10 +55,11 @@ from vllm.model_executor.layers.quantization.gptq_marlin import (
55 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
56 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
57 from vllm.multimodal import MULTIMODAL_REGISTRY
57 -from vllm.multimodal.inputs import MultiModalDataDict, NestedTensors
58 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalDataItems,
59 + MultiModalFieldConfig, MultiModalKwargs,
60 + NestedTensors)
61 from vllm.multimodal.processing import (BaseMultiModalProcessor,
59 - MultiModalDataItems, ProcessorInputs,
60 - PromptReplacement)
62 + ProcessorInputs, PromptReplacement)
63 from vllm.platforms import _Backend
64 from vllm.sequence import IntermediateTensors
65 from vllm.transformers_utils.config import uses_mrope
@@ -229,9 +231,9 @@ class Qwen2VisionAttention(nn.Module):
231
232 def __init__(
233 self,
232 - embed_dim: Optional[int] = None,
233 - num_heads: Optional[int] = None,
234 - projection_size: Optional[int] = None,
234 + embed_dim: int,
235 + num_heads: int,
236 + projection_size: int,
237 quant_config: Optional[QuantizationConfig] = None,
238 prefix: str = "",
239 ) -> None:
@@ -264,7 +266,7 @@ class Qwen2VisionAttention(nn.Module):
266 self,
267 x: torch.Tensor,
268 cu_seqlens: torch.Tensor,
267 - rotary_pos_emb: torch.Tensor = None,
269 + rotary_pos_emb: torch.Tensor,
270 ) -> torch.Tensor:
271 # [s, b, c] --> [s, b, head * 3 * head_dim]
272 x, _ = self.qkv(x)
@@ -347,7 +349,7 @@ class Qwen2VisionBlock(nn.Module):
349 num_heads: int,
350 mlp_ratio: float,
351 act_layer: Type[nn.Module] = QuickGELU,
350 - norm_layer: Type[nn.Module] = None,
352 + norm_layer: Optional[Callable[[int], nn.Module]] = None,
353 quant_config: Optional[QuantizationConfig] = None,
354 prefix: str = "",
355 ) -> None:
@@ -384,7 +386,7 @@ class Qwen2VisionPatchEmbed(nn.Module):
386 self,
387 patch_size: int = 14,
388 temporal_patch_size: int = 2,
387 - in_chans: int = 3,
389 + in_channels: int = 3,
390 embed_dim: int = 1152,
391 ) -> None:
392 super().__init__()
@@ -392,8 +394,8 @@ class Qwen2VisionPatchEmbed(nn.Module):
394 self.temporal_patch_size = temporal_patch_size
395 self.embed_dim = embed_dim
396
395 - kernel_size = [temporal_patch_size, patch_size, patch_size]
396 - self.proj = nn.Conv3d(in_chans,
397 + kernel_size = (temporal_patch_size, patch_size, patch_size)
398 + self.proj = nn.Conv3d(in_channels,
399 embed_dim,
400 kernel_size=kernel_size,
401 stride=kernel_size,
@@ -413,7 +415,7 @@ class Qwen2VisionPatchMerger(nn.Module):
415 self,
416 d_model: int,
417 context_dim: int,
416 - norm_layer: Type[nn.Module] = None,
418 + norm_layer: Optional[Callable[[int], nn.Module]] = None,
419 spatial_merge_size: int = 2,
420 quant_config: Optional[QuantizationConfig] = None,
421 prefix: str = "",
@@ -489,15 +491,15 @@ class Qwen2VisionTransformer(nn.Module):
491 ) -> None:
492 super().__init__()
493
492 - patch_size: int = vision_config.patch_size
493 - temporal_patch_size: int = vision_config.temporal_patch_size
494 - spatial_merge_size: int = vision_config.spatial_merge_size
495 - in_chans: int = vision_config.in_chans
496 - hidden_size: int = vision_config.hidden_size
497 - embed_dim: int = vision_config.embed_dim
498 - depth: int = vision_config.depth
499 - num_heads: int = vision_config.num_heads
500 - mlp_ratio: float = vision_config.mlp_ratio
494 + patch_size = vision_config.patch_size
495 + temporal_patch_size = vision_config.temporal_patch_size
496 + spatial_merge_size = vision_config.spatial_merge_size
497 + in_channels = vision_config.in_channels
498 + hidden_size = vision_config.hidden_size
499 + embed_dim = vision_config.embed_dim
500 + depth = vision_config.depth
501 + num_heads = vision_config.num_heads
502 + mlp_ratio = vision_config.mlp_ratio
503
504 self.spatial_merge_size = spatial_merge_size
505 self.num_heads = num_heads
@@ -506,7 +508,7 @@ class Qwen2VisionTransformer(nn.Module):
508 self.patch_embed = Qwen2VisionPatchEmbed(
509 patch_size=patch_size,
510 temporal_patch_size=temporal_patch_size,
509 - in_chans=in_chans,
511 + in_channels=in_channels,
512 embed_dim=embed_dim,
513 )
514
@@ -733,8 +735,12 @@ class Qwen2VLMultiModalDataItems(MultiModalDataItems):
735 if k == "video":
736 # Special case since even a single item can be a list
737 multi_data[k] = ( # type: ignore[index]
736 - v if (isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
737 - or is_list_of(v, list)) else [v]
738 + v if (
739 + isinstance(v, (dict, torch.Tensor)) # type: ignore[assignment]
740 + or is_list_of(v, list)
741 + or isinstance(v[0], (np.ndarray, torch.Tensor))
742 + and v[0].ndim == 4
743 + ) else [v]
744 )
745 elif k in ("image", "audio"):
746 multi_data[k] = ( # type: ignore[index]
@@ -754,6 +760,12 @@ class Qwen2VLMultiModalDataItems(MultiModalDataItems):
760 for m, items in self.items()
761 }
762
763 + def has_embedding_inputs(self) -> bool:
764 + return any(
765 + isinstance(items, dict) or any(
766 + isinstance(item, torch.Tensor) for item in items)
767 + for items in self.values())
768 +
769
770 class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
771
@@ -784,7 +796,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
796
797 return hf_processor
798
787 - def _get_processor_data(
799 + def _get_hf_mm_data(
800 self,
801 mm_items: MultiModalDataItems,
802 ) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -805,7 +817,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
817 and v[0].ndim == 2):
818 # Pass through embedding inputs (multi)
819 passthrough_data[f"{k}_embeds"] = v
808 - else:
820 + elif len(v) > 0:
821 # Map keys to plural form, e.g.: image -> images
822 processor_data[f"{k}s"] = v
823 else:
@@ -816,8 +828,8 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
828 def _get_prompt_replacements(
829 self,
830 mm_items: MultiModalDataItems,
819 - hf_inputs: BatchFeature,
820 - mm_processor_kwargs: Mapping[str, object],
831 + hf_processor_mm_kwargs: Mapping[str, object],
832 + out_mm_kwargs: MultiModalKwargs,
833 ) -> list[PromptReplacement]:
834 hf_processor = self._get_hf_processor()
835 image_processor = _get_image_processor(hf_processor)
@@ -831,7 +843,9 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
843 merge_length = image_processor.merge_size**2
844
845 def get_replacement_qwen2vl(item_idx: int, modality: str):
834 - grid_thw = hf_inputs[f"{modality}_grid_thw"][item_idx]
846 + grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
847 + assert isinstance(grid_thw, torch.Tensor)
848 +
849 num_tokens = grid_thw.prod() // merge_length
850 return placeholder[modality] * num_tokens
851
@@ -844,11 +858,40 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
858 ) for modality in ("image", "video")
859 ]
860
861 + def _get_mm_fields_config(
862 + self,
863 + hf_inputs: BatchFeature,
864 + hf_processor_mm_kwargs: Mapping[str, object],
865 + ) -> Mapping[str, MultiModalFieldConfig]:
866 + image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
867 + image_slice_idxs = [0] + image_grid_thw.prod(-1).cumsum_(0).tolist()
868 + image_slices = [
869 + slice(image_slice_idxs[i], image_slice_idxs[i + 1])
870 + for i in range(len(image_grid_thw))
871 + ]
872 +
873 + video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
874 + video_slice_idxs = [0] + video_grid_thw.prod(-1).cumsum_(0).tolist()
875 + video_slices = [
876 + slice(video_slice_idxs[i], video_slice_idxs[i + 1])
877 + for i in range(len(video_grid_thw))
878 + ]
879 +
880 + return dict(
881 + pixel_values=MultiModalFieldConfig.flat("image", image_slices),
882 + image_embeds=MultiModalFieldConfig.flat("image", image_slices),
883 + image_grid_thw=MultiModalFieldConfig.batched("image"),
884 + pixel_values_videos=MultiModalFieldConfig.flat(
885 + "video", video_slices),
886 + video_embeds=MultiModalFieldConfig.flat("video", video_slices),
887 + video_grid_thw=MultiModalFieldConfig.batched("video"),
888 + )
889 +
890 def _get_dummy_mm_inputs(
891 self,
892 mm_counts: Mapping[str, int],
893 ) -> ProcessorInputs:
851 - num_images = mm_counts["image"]
894 + num_images = mm_counts.get("image", 0)
895 hf_processor = self._get_hf_processor()
896 image_token: str = hf_processor.image_token
897 image_processor = _get_image_processor(hf_processor)
@@ -869,7 +912,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
912 return ProcessorInputs(
913 prompt_text=image_token * num_images,
914 mm_data=data,
872 - mm_processor_kwargs={},
915 )
916
917
@@ -950,9 +992,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
992 return None
993 return quant_config
994
953 - def _validate_and_reshape_mm_tensor(self,
954 - mm_input: Union[torch.Tensor,
955 - List[torch.Tensor]],
995 + def _validate_and_reshape_mm_tensor(self, mm_input: object,
996 name: str) -> torch.Tensor:
997 if not isinstance(mm_input, (torch.Tensor, list)):
998 raise ValueError(f"Incorrect type of {name}. "
@@ -962,7 +1002,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1002 return mm_input
1003 if mm_input.ndim != 3:
1004 raise ValueError(f"{name} should be 2D or batched 3D tensor. "
965 - f"Got ndim: {mm_input.ndim}")
1005 + f"Got ndim: {mm_input.ndim} "
1006 + f"(shape={mm_input.shape})")
1007 return torch.concat(list(mm_input))
1008 else:
1009 return torch.concat(mm_input)
vllm/model_executor/models/ultravox.py
+51 -25
@@ -23,10 +23,11 @@ from vllm.model_executor.layers.layernorm import RMSNorm
23 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
24 from vllm.model_executor.model_loader.loader import DefaultModelLoader
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 -from vllm.multimodal import MULTIMODAL_REGISTRY, NestedTensors
26 +from vllm.multimodal import MULTIMODAL_REGISTRY
27 +from vllm.multimodal.inputs import (MultiModalDataItems, MultiModalFieldConfig,
28 + MultiModalKwargs, NestedTensors)
29 from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 - MultiModalDataItems, ProcessorInputs,
29 - PromptReplacement)
30 + ProcessorInputs, PromptReplacement)
31 from vllm.sequence import IntermediateTensors
32 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
33 from vllm.utils import is_list_of
@@ -72,11 +73,19 @@ def get_ultravox_max_audio_tokens(ctx: InputContext):
73
74 class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
75
76 + def _get_hf_processor(
77 + self,
78 + *,
79 + # Ignored in initialization
80 + sampling_rate: Optional[int] = None,
81 + ) -> ProcessorMixin:
82 + return self.ctx.get_hf_processor()
83 +
84 def _get_feature_extractor(self) -> WhisperFeatureExtractor:
85 hf_processor = self._get_hf_processor()
86 return hf_processor.audio_processor.feature_extractor # type: ignore
87
79 - def _get_processor_data(
88 + def _get_hf_mm_data(
89 self,
90 mm_items: MultiModalDataItems,
91 ) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -84,33 +93,41 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
93 feature_extractor = self._get_feature_extractor()
94 mm_items.resample_audios(feature_extractor.sampling_rate)
95
87 - return super()._get_processor_data(mm_items)
96 + return super()._get_hf_mm_data(mm_items)
97
98 def _call_hf_processor(
99 self,
91 - hf_processor: ProcessorMixin,
100 prompt: str,
93 - processor_data: Mapping[str, object],
94 - mm_processor_kwargs: Mapping[str, object],
101 + mm_data: Mapping[str, object],
102 + mm_kwargs: Mapping[str, object],
103 ) -> BatchFeature:
96 - processor_data = dict(processor_data)
97 - audios = processor_data.pop("audios", [])
104 + # Text-only input not supported in composite processor
105 + if not mm_data:
106 + tokenizer = self._get_tokenizer()
107 +
108 + prompt_ids = tokenizer.encode(
109 + prompt,
110 + add_special_tokens=False, # type: ignore
111 + )
112 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
113 +
114 + mm_data = dict(mm_data)
115 + audios = mm_data.pop("audios", [])
116
117 if not audios:
118 return super()._call_hf_processor(
101 - hf_processor,
119 prompt=prompt,
103 - processor_data=processor_data,
104 - mm_processor_kwargs=mm_processor_kwargs,
120 + mm_data=mm_data,
121 + mm_kwargs=mm_kwargs,
122 )
123
124 feature_extractor = self._get_feature_extractor()
108 - mm_processor_kwargs = dict(
109 - **mm_processor_kwargs,
125 + mm_kwargs = dict(
126 + **mm_kwargs,
127 sampling_rate=feature_extractor.sampling_rate,
128 )
129
113 - # Already resampled by _get_processor_data
130 + # Already resampled by _get_hf_mm_data
131 assert is_list_of(audios, np.ndarray)
132
133 # Ultravox processor doesn't support multiple inputs,
@@ -119,13 +136,12 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
136 shared_outputs = {}
137 for audio in audios:
138 # NOTE: Ultravox processor accepts "audio" instead of "audios"
122 - item_processor_data = dict(**processor_data, audio=audio)
139 + item_processor_data = dict(**mm_data, audio=audio)
140
141 item_outputs = super()._call_hf_processor(
125 - hf_processor,
142 prompt=prompt,
127 - processor_data=item_processor_data,
128 - mm_processor_kwargs=mm_processor_kwargs,
143 + mm_data=item_processor_data,
144 + mm_kwargs=mm_kwargs,
145 )
146
147 audio_features.append(item_outputs.pop("audio_values")[0])
@@ -139,17 +155,28 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
155 )
156 return BatchFeature(combined_outputs)
157
158 + def _get_mm_fields_config(
159 + self,
160 + hf_inputs: BatchFeature,
161 + hf_processor_mm_kwargs: Mapping[str, object],
162 + ) -> Mapping[str, MultiModalFieldConfig]:
163 + return dict(
164 + audio_features=MultiModalFieldConfig.batched("audio"),
165 + audio_token_len=MultiModalFieldConfig.batched("audio"),
166 + audio_embeds=MultiModalFieldConfig.batched("audio"),
167 + )
168 +
169 def _get_prompt_replacements(
170 self,
171 mm_items: MultiModalDataItems,
145 - hf_inputs: BatchFeature,
146 - mm_processor_kwargs: Mapping[str, object],
172 + hf_processor_mm_kwargs: Mapping[str, object],
173 + out_mm_kwargs: MultiModalKwargs,
174 ) -> list[PromptReplacement]:
175 hf_processor = self._get_hf_processor()
176 placeholder = hf_processor.audio_token_replacement # type: ignore
177
178 def get_replacement_ultravox(item_idx: int):
152 - audio_token_len = hf_inputs["audio_token_len"][item_idx]
179 + audio_token_len = out_mm_kwargs["audio_token_len"][item_idx]
180 return placeholder * audio_token_len
181
182 return [
@@ -168,14 +195,13 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
195 sampling_rate = feature_extractor.sampling_rate
196 audio_len = feature_extractor.chunk_length * sampling_rate
197
171 - audio_count = mm_counts["audio"]
198 + audio_count = mm_counts.get("audio", 0)
199 audio = np.zeros(audio_len)
200 data = {"audio": [audio] * audio_count}
201
202 return ProcessorInputs(
203 prompt_text="<|audio|>" * audio_count,
204 mm_data=data,
178 - mm_processor_kwargs={},
205 )
206
207
vllm/multimodal/base.py
+23 -21
@@ -297,35 +297,37 @@ class MultiModalPlaceholderMap:
297 ``MultiModalPlaceholderMap`` that relates the multi-modal embedding
298 vectors to their corresponding placeholders.
299
300 - Consider the following scenarios:
300 + Examples:
301
302 - Prompt: |AAAA BBBB What's in these images?|
303 - Positions: |.................................|
302 + .. code-block::
303
305 - images = [A, B]
306 - src_ranges = [(0, 4), (4, 8)]
307 - dest_ranges = [(0, 4), (5, 9)]
304 + Prompt: |AAAA BBBB What's in these images?|
305 + Positions: |.................................|
306
309 - Prompt: |AAAA BBBB What's in these images?|
310 - Positions: | ..... |
307 + images = [A, B]
308 + src_ranges = [(0, 4), (4, 8)]
309 + dest_ranges = [(0, 4), (5, 9)]
310
312 - images = [A, B]
313 - src_ranges = [(2, 4), (4, 6)]
314 - dest_ranges = [(0, 2), (3, 5)]
311 + Prompt: |AAAA BBBB What's in these images?|
312 + Positions: | ..... |
313
316 - Prompt: |AAAA BBBB What's in these images?|
317 - Positions: | ......... |
314 + images = [A, B]
315 + src_ranges = [(2, 4), (4, 6)]
316 + dest_ranges = [(0, 2), (3, 5)]
317
319 - images = [B]
320 - src_ranges = [(0, 4)]
321 - dest_ranges = [(0, 4)]
318 + Prompt: |AAAA BBBB What's in these images?|
319 + Positions: | ......... |
320
323 - Prompt: |AAAA BBBB What's in these images?|
324 - Positions: | .......................|
321 + images = [B]
322 + src_ranges = [(0, 4)]
323 + dest_ranges = [(0, 4)]
324
326 - images = []
327 - src_ranges = []
328 - dest_ranges = []
325 + Prompt: |AAAA BBBB What's in these images?|
326 + Positions: | .......................|
327 +
328 + images = []
329 + src_ranges = []
330 + dest_ranges = []
331 """
332 seq_mm_data = seq_group.multi_modal_data
333 seq_mm_placeholders = seq_group.multi_modal_placeholders
vllm/multimodal/inputs.py
+423 -15
@@ -1,12 +1,16 @@
1 +from abc import ABC, abstractmethod
2 from collections import UserDict, defaultdict
2 -from typing import (Any, Dict, List, Literal, Mapping, Sequence, Tuple,
3 - TypedDict, TypeVar, Union, cast, final)
3 +from collections.abc import Mapping, Sequence
4 +from dataclasses import dataclass
5 +from typing import (Any, Literal, NamedTuple, TypedDict, TypeVar, Union, cast,
6 + final)
7
8 import numpy as np
9 import torch
10 import torch.types
11 from PIL.Image import Image
9 -from typing_extensions import NotRequired, TypeAlias
12 +from transformers import BatchFeature
13 +from typing_extensions import NotRequired, TypeAlias, assert_never
14
15 from vllm.utils import JSONTree, is_list_of, json_map_leaves
16
@@ -44,7 +48,7 @@ item, which can be passed to a HuggingFace :code:`AudioProcessor`.
48 """
49 # yapf: enable
50
47 -MultiModalData: TypeAlias = Union[_T, List[_T]]
51 +MultiModalData: TypeAlias = Union[_T, list[_T]]
52 """
53 Either a single data item, or a list of data items.
54
@@ -79,13 +83,135 @@ Note:
83 """
84
85
86 +class ImageSize(NamedTuple):
87 + width: int
88 + height: int
89 +
90 +
91 +class MultiModalDataItems(UserDict[str, list[Any]]):
92 + """
93 + As :class:`MultiModalDataDict`, but normalized such that each entry
94 + corresponds to a list.
95 + """
96 +
97 + @staticmethod
98 + def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
99 + """
100 + Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
101 + """
102 + multi_data = MultiModalDataItems()
103 +
104 + for k, v in data.items():
105 + # TODO: Make a separate modality for embedding inputs
106 + # to avoid confusion
107 + # yapf: disable
108 + if k == "video":
109 + # Special case since even a single item can be a list
110 + multi_data[k] = ( # type: ignore[index]
111 + v if (
112 + isinstance(v, torch.Tensor)
113 + or is_list_of(v, list)
114 + or isinstance(v[0], (np.ndarray, torch.Tensor))
115 + and v[0].ndim == 4
116 + ) else [v]
117 + )
118 + elif k in ("image", "audio"):
119 + multi_data[k] = ( # type: ignore[index]
120 + v if isinstance(v, (torch.Tensor, list)) else [v]
121 + )
122 + else:
123 + multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
124 + # yapf: enable
125 +
126 + return multi_data
127 +
128 + # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
129 + # `self.images` doesn't update this dictionary, which may be confusing
130 + # We annotate the getter methods as `Sequence` to prevent others from
131 + # trying to update the list in this way
132 + @property
133 + def images(self) -> Sequence[ImageItem]:
134 + return self.get("image", [])
135 +
136 + @property
137 + def videos(self) -> Sequence[VideoItem]:
138 + return self.get("video", [])
139 +
140 + @property
141 + def audios(self) -> Sequence[AudioItem]:
142 + return self.get("audio", [])
143 +
144 + def get_item_counts(self) -> Mapping[str, int]:
145 + return {m: len(items) for m, items in self.items()}
146 +
147 + def has_embedding_inputs(self) -> bool:
148 + return any(
149 + any(isinstance(item, torch.Tensor) for item in items)
150 + for items in self.values())
151 +
152 + def get_image_size(self, item_idx: int) -> ImageSize:
153 + image = self.images[item_idx]
154 +
155 + if isinstance(image, Image):
156 + return ImageSize(*image.size)
157 + if isinstance(image, (np.ndarray, torch.Tensor)):
158 + _, h, w = image.shape
159 + return ImageSize(w, h)
160 +
161 + assert_never(image)
162 +
163 + def get_audio_with_sr(
164 + self,
165 + item_idx: int,
166 + *,
167 + default_sr: float,
168 + ) -> tuple[np.ndarray, float]:
169 + audio = self.audios[item_idx]
170 +
171 + if isinstance(audio, tuple):
172 + return audio
173 + if isinstance(audio, list):
174 + return np.array(audio), default_sr
175 + if isinstance(audio, np.ndarray):
176 + return audio, default_sr
177 +
178 + assert_never(audio)
179 +
180 + def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
181 + """
182 + If :code:`drop_sr=True`, the audio items in this dictionary are updated
183 + to be NumPy arrays which implicitly means that their sampling rate is
184 + the same as the model's expected sampling rate; otherwise, they remain
185 + as :code:`(audio, new_sr)` tuples.
186 + """
187 + # Avoid circular import
188 + from .audio import resample_audio
189 +
190 + if not self.audios:
191 + return
192 +
193 + new_audios = []
194 + for item_idx in range(len(self.audios)):
195 + audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
196 + audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
197 +
198 + new_audios.append(audio if drop_sr else (audio, new_sr))
199 +
200 + self["audio"] = new_audios
201 +
202 +
203 class PlaceholderRange(TypedDict):
204 """
205 Placeholder location information for multi-modal data.
206
86 - For example:
87 - Prompt: AAAA BBBB What is in these images?
207 + Example:
208 +
209 + Prompt: :code:`AAAA BBBB What is in these images?`
210 +
211 Images A and B will have:
212 +
213 + .. code-block::
214 +
215 A: { "offset": 0, "length": 4 }
216 B: { "offset": 5, "length": 4 }
217 """
@@ -97,25 +223,256 @@ class PlaceholderRange(TypedDict):
223 """The length of the placeholder."""
224
225
100 -NestedTensors = Union[List["NestedTensors"], List[torch.Tensor], torch.Tensor,
101 - Tuple[torch.Tensor, ...]]
226 +NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,
227 + tuple[torch.Tensor, ...]]
228 """
229 Uses a list instead of a tensor if the dimensions of each element do not match.
230 """
231
106 -BatchedTensorInputs: TypeAlias = Dict[str, NestedTensors]
232 +
233 +def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
234 + """Equality check between :data:`NestedTensors` objects."""
235 + if isinstance(a, torch.Tensor):
236 + return isinstance(b, torch.Tensor) and bool((a == b).all().item())
237 + elif isinstance(b, torch.Tensor):
238 + return isinstance(a, torch.Tensor) and bool((b == a).all().item())
239 +
240 + if isinstance(a, list):
241 + return (isinstance(b, list)
242 + and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)))
243 + if isinstance(b, list):
244 + return (isinstance(a, list)
245 + and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)))
246 +
247 + # Both a and b are scalars
248 + return a == b
249 +
250 +
251 +BatchedTensorInputs: TypeAlias = Mapping[str, NestedTensors]
252 """
253 A dictionary containing nested tensors which have been batched via
254 :meth:`MultiModalKwargs.batch`.
255 """
256
257
258 +@dataclass(frozen=True)
259 +class MultiModalFieldItem:
260 + """
261 + Contains metadata and data in :class:`MultiModalKwargs`
262 + corresponding to a data item in :class:`MultiModalDataItems`.
263 + """
264 + field: "BaseMultiModalField"
265 + data: NestedTensors
266 +
267 + def __eq__(self, other: object) -> bool:
268 + if not isinstance(other, self.__class__):
269 + return False
270 +
271 + return (self.field == other.field
272 + and nested_tensors_equal(self.data, other.data))
273 +
274 +
275 +@dataclass(frozen=True)
276 +class BaseMultiModalField(ABC):
277 + """Abstract base class for a field in :class:`MultiModalKwargs`."""
278 + key: str
279 + modality: str
280 +
281 + @abstractmethod
282 + def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
283 + raise NotImplementedError
284 +
285 + def _build_item(self, data: NestedTensors) -> MultiModalFieldItem:
286 + return MultiModalFieldItem(self, data)
287 +
288 + def reduce(self, batch: list[MultiModalFieldItem]) -> MultiModalFieldItem:
289 + """Merge multiple instances of :class:`MultiModalFieldItem` together."""
290 + fields = [item.field for item in batch]
291 + if len(set(fields)) > 1:
292 + raise ValueError(f"Cannot merge different {fields=}")
293 +
294 + data = self._reduce_data([item.data for item in batch])
295 +
296 + return self._build_item(data)
297 +
298 +
299 +@dataclass(frozen=True)
300 +class MultiModalBatchedField(BaseMultiModalField):
301 + """
302 + A :class:`BaseMultiModalField` implementation where an item is obtained by
303 + directly indexing into the first dimension of the underlying data.
304 + """
305 +
306 + def build_items(self, batch: NestedTensors) -> list[MultiModalFieldItem]:
307 + return [self._build_item(item) for item in batch]
308 +
309 + def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
310 + if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
311 + first_shape = batch[0].shape
312 + if all(item.shape == first_shape for item in batch):
313 + return torch.stack(batch)
314 +
315 + return batch
316 +
317 +
318 +@dataclass(frozen=True)
319 +class MultiModalFlatField(BaseMultiModalField):
320 + """
321 + A :class:`BaseMultiModalField` implementation where an item is obtained by
322 + slicing along the first dimension of the underlying data.
323 + """
324 +
325 + def build_items(
326 + self,
327 + batch: NestedTensors,
328 + slices: Sequence[slice],
329 + ) -> list[MultiModalFieldItem]:
330 + return [self._build_item(batch[slice_]) for slice_ in slices]
331 +
332 + def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
333 + if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
334 + first_shape = batch[0].shape
335 + if all(item.shape[1:] == first_shape[1:] for item in batch):
336 + return torch.concat(batch)
337 +
338 + return [elem for item in batch for elem in item]
339 +
340 +
341 +class MultiModalFieldConfig:
342 +
343 + @staticmethod
344 + def batched(modality: str):
345 + return MultiModalFieldConfig(
346 + field_cls=MultiModalBatchedField,
347 + modality=modality,
348 + )
349 +
350 + @staticmethod
351 + def flat(modality: str, slices: Sequence[slice]):
352 + return MultiModalFieldConfig(
353 + field_cls=MultiModalFlatField,
354 + modality=modality,
355 + slices=slices,
356 + )
357 +
358 + def __init__(
359 + self,
360 + field_cls: type[BaseMultiModalField],
361 + modality: str,
362 + **field_config: Any,
363 + ) -> None:
364 + super().__init__()
365 +
366 + self._field_cls = field_cls
367 + self._modality = modality
368 + self._field_config = field_config
369 +
370 + def build_items(
371 + self,
372 + key: str,
373 + batch: NestedTensors,
374 + ) -> list[MultiModalFieldItem]:
375 + field = self._field_cls(key=key, modality=self._modality)
376 + return field.build_items(batch, **self._field_config) # type: ignore
377 +
378 +
379 class MultiModalKwargs(UserDict[str, NestedTensors]):
380 """
381 A dictionary that represents the keyword arguments to
382 :meth:`~torch.nn.Module.forward`.
383 +
384 + The metadata :code:`items_by_key` defines how to split batched keyword
385 + arguments corresponding to each data item in :class:`MultiModalDataItems`:
386 +
387 + - For a keyword argument, we can access the :code:`i` th item in the batch
388 + via :code:`items_by_key[key][i]`.
389 + - We can gather the keyword arguments belonging to a modality by finding
390 + the keys with items that belong to that modality, then accessing
391 + the :code:`i` th item in the batch for each such key.
392 +
393 + Example:
394 +
395 + .. code-block:: python
396 +
397 + # All items belong to the "image" modality
398 + items_by_key={
399 + "pixel_values": [a, b, c, d], # "image" modality
400 + "image_grid_thw": [e, f, g, h], # "image" modality
401 + "pixel_values_video": [h, i, j], # "video" modality
402 + "video_grid_thw": [k, l, m], # "video" modality
403 + }
404 +
405 + - The keyword arguments belonging to the first image are
406 + :code:`{"pixel_values": a, "image_grid_thw": e}`.
407 + - The keyword arguments belonging to the second video are
408 + :code:`{"pixel_values_video": i, "video_grid_thw": l}`.
409 """
410
411 + @staticmethod
412 + def from_hf_inputs(
413 + hf_inputs: BatchFeature,
414 + config_by_key: Mapping[str, MultiModalFieldConfig],
415 + *,
416 + enable_sanity_checks: bool = False,
417 + ):
418 + # NOTE: This skips fields in `hf_inputs` that are not in `config_by_key`
419 + # We assume that those fields are not used in vLLM
420 + items_by_key = {
421 + key: config.build_items(key, batch)
422 + for key, config in config_by_key.items()
423 + if (batch := hf_inputs.get(key)) is not None
424 + }
425 +
426 + return MultiModalKwargs.from_items_by_key(
427 + items_by_key,
428 + enable_sanity_checks=enable_sanity_checks,
429 + )
430 +
431 + @staticmethod
432 + def from_items_by_key(
433 + items_by_key: Mapping[str, list[MultiModalFieldItem]],
434 + *,
435 + enable_sanity_checks: bool = False,
436 + ) -> "MultiModalKwargs":
437 + data = {
438 + key: items[0].field.reduce(items).data
439 + for key, items in items_by_key.items()
440 + }
441 +
442 + return MultiModalKwargs(data,
443 + items_by_key=items_by_key,
444 + enable_sanity_checks=enable_sanity_checks)
445 +
446 + def __init__(
447 + self,
448 + data: Mapping[str, NestedTensors],
449 + *,
450 + items_by_key: Mapping[str, list[MultiModalFieldItem]] = {},
451 + enable_sanity_checks: bool = False,
452 + ) -> None:
453 + super().__init__(data)
454 +
455 + # Shallow copy to avoid footgun in case a defaultdict is passed in
456 + self._items_by_key = dict(items_by_key)
457 +
458 + keys_by_modality = defaultdict[str, set[str]](set)
459 + for key, items in items_by_key.items():
460 + for item in items:
461 + keys_by_modality[item.field.modality].add(key)
462 +
463 + self._keys_by_modality = dict(keys_by_modality)
464 +
465 + if enable_sanity_checks:
466 + for modality, keys in keys_by_modality.items():
467 + items_in_modality = {k: items_by_key[k] for k in keys}
468 + batch_sizes = {k: len(v) for k, v in items_in_modality.items()}
469 + batch_size = next(iter(batch_sizes.values()), 0)
470 + assert all(bs == batch_size
471 + for bs in batch_sizes.values()), dict(
472 + modality=modality,
473 + batch_sizes=batch_sizes,
474 + items_by_key=items_by_key)
475 +
476 @staticmethod
477 def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
478 """
@@ -139,7 +496,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
496 # Only tensors (not lists) can be stacked.
497 return stacked
498
142 - tensors_ = cast(List[torch.Tensor], stacked)
499 + tensors_ = cast(list[torch.Tensor], stacked)
500 if any(t.shape != tensors_[0].shape for t in tensors_):
501 # The tensors have incompatible shapes and can't be stacked.
502 return tensors_
@@ -147,7 +504,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
504 return torch.stack(tensors_)
505
506 @staticmethod
150 - def batch(inputs_list: List["MultiModalKwargs"]) -> BatchedTensorInputs:
507 + def batch(inputs_list: list["MultiModalKwargs"]) -> BatchedTensorInputs:
508 """
509 Batch multiple inputs together into a dictionary.
510
@@ -162,7 +519,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
519
520 # We need to consider the case where each item in the batch
521 # contains different modalities (i.e. different keys).
165 - item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
522 + item_lists = defaultdict[str, list[NestedTensors]](list)
523
524 for inputs in inputs_list:
525 for k, v in inputs.items():
@@ -188,6 +545,57 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
545
546 return cast(BatchedTensorInputs, json_mapped)
547
548 + def __eq__(self, other: object) -> bool:
549 + if not isinstance(other, self.__class__):
550 + return False
551 + if self._items_by_key != other._items_by_key:
552 + return False
553 +
554 + ks = self.keys()
555 + return (ks == other.keys()
556 + and all(nested_tensors_equal(self[k], other[k]) for k in ks))
557 +
558 + def get_item(self, key: str, item_index: int) -> MultiModalFieldItem:
559 + return self._items_by_key[key][item_index]
560 +
561 + def get_items_by_modality(
562 + self,
563 + modality: str,
564 + item_index: int,
565 + ) -> Mapping[str, MultiModalFieldItem]:
566 + """
567 + Get the keyword arguments corresponding to an item identified by
568 + its modality and index.
569 + """
570 + keys_to_gather = self._keys_by_modality[modality]
571 +
572 + return {
573 + key: self.get_item(key, item_index)
574 + for key in keys_to_gather if key in self
575 + }
576 +
577 + @staticmethod
578 + def from_items_by_modality(
579 + items_by_modality: Mapping[str, list[Mapping[str,
580 + MultiModalFieldItem]]],
581 + *,
582 + enable_sanity_checks: bool = False,
583 + ) -> "MultiModalKwargs":
584 + """
585 + Construct a new :class:`MultiModalKwargs` from multiple items returned
586 + by :meth:`get_fields_by_modality`.
587 + """
588 + items_by_key = defaultdict[str, list[MultiModalFieldItem]](list)
589 + for fields in items_by_modality.values():
590 + for field in fields:
591 + for k, v in field.items():
592 + items_by_key[k].append(v)
593 +
594 + return MultiModalKwargs.from_items_by_key(
595 + items_by_key,
596 + enable_sanity_checks=enable_sanity_checks,
597 + )
598 +
599
600 MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]
601 """
@@ -207,16 +615,16 @@ class MultiModalInputsV2(TypedDict):
615 prompt: str
616 """The processed prompt text."""
617
210 - prompt_token_ids: List[int]
618 + prompt_token_ids: list[int]
619 """The processed token IDs which includes placeholder tokens."""
620
213 - token_type_ids: NotRequired[List[int]]
621 + token_type_ids: NotRequired[list[int]]
622 """The token type IDs of the prompt."""
623
624 mm_kwargs: MultiModalKwargs
625 """Keyword arguments to be directly passed to the model after batching."""
626
219 - mm_hashes: NotRequired[List[str]]
627 + mm_hashes: NotRequired[list[str]]
628 """The hashes of the multi-modal data."""
629
630 mm_placeholders: MultiModalPlaceholderDict
vllm/multimodal/processing.py
+347 -169
@@ -1,6 +1,6 @@
1 +import pickle
2 import re
3 from abc import ABC, abstractmethod
3 -from collections import UserDict
4 from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
5 from dataclasses import dataclass, field
6 from functools import lru_cache
@@ -8,19 +8,18 @@ from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
8
9 import numpy as np
10 import torch
11 +from blake3 import blake3
12 from PIL.Image import Image
13 from transformers import BatchFeature, ProcessorMixin
13 -from typing_extensions import assert_never
14
15 from vllm.inputs import DummyData, InputProcessingContext
16 from vllm.logger import init_logger
17 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
18 -from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
18 +from vllm.utils import LRUCache, flatten_2d_lists, full_groupby, is_list_of
19
20 -from .audio import resample_audio
21 -from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
22 - MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
23 - VideoItem)
20 +from .inputs import (MultiModalDataDict, MultiModalDataItems,
21 + MultiModalFieldConfig, MultiModalFieldItem,
22 + MultiModalInputsV2, MultiModalKwargs, PlaceholderRange)
23
24 logger = init_logger(__name__)
25
@@ -201,111 +200,6 @@ class _BoundPromptReplacement:
200 return bound_replacement
201
202
204 -class ImageSize(NamedTuple):
205 - width: int
206 - height: int
207 -
208 -
209 -class MultiModalDataItems(UserDict[str, list[Any]]):
210 - """
211 - As :class:`MultiModalDataDict`, but normalized such that each entry
212 - corresponds to a list.
213 - """
214 -
215 - @staticmethod
216 - def from_dict(data: MultiModalDataDict) -> "MultiModalDataItems":
217 - """
218 - Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`.
219 - """
220 - multi_data = MultiModalDataItems()
221 -
222 - for k, v in data.items():
223 - # TODO: Make a separate modality for embedding inputs
224 - # to avoid confusion
225 - # yapf: disable
226 - if k == "video":
227 - # Special case since even a single item can be a list
228 - multi_data[k] = ( # type: ignore[index]
229 - v if (isinstance(v, torch.Tensor)
230 - or is_list_of(v, list)) else [v]
231 - )
232 - elif k in ("image", "audio"):
233 - multi_data[k] = ( # type: ignore[index]
234 - v if isinstance(v, (torch.Tensor, list)) else [v]
235 - )
236 - else:
237 - multi_data[k] = v if isinstance(v, list) else [v] # type: ignore[index]
238 - # yapf: enable
239 -
240 - return multi_data
241 -
242 - # NOTE: When a field (e.g. `images`) doesn't exist, directly appending to
243 - # `self.images` doesn't update this dictionary, which may be confusing
244 - # We annotate the getter methods as `Sequence` to prevent others from
245 - # trying to update the list in this way
246 - @property
247 - def images(self) -> Sequence[ImageItem]:
248 - return self.get("image", [])
249 -
250 - @property
251 - def videos(self) -> Sequence[VideoItem]:
252 - return self.get("video", [])
253 -
254 - @property
255 - def audios(self) -> Sequence[AudioItem]:
256 - return self.get("audio", [])
257 -
258 - def get_item_counts(self) -> Mapping[str, int]:
259 - return {m: len(items) for m, items in self.items()}
260 -
261 - def get_image_size(self, item_idx: int) -> ImageSize:
262 - image = self.images[item_idx]
263 -
264 - if isinstance(image, Image):
265 - return ImageSize(*image.size)
266 - if isinstance(image, (np.ndarray, torch.Tensor)):
267 - _, h, w = image.shape
268 - return ImageSize(w, h)
269 -
270 - assert_never(image)
271 -
272 - def get_audio_with_sr(
273 - self,
274 - item_idx: int,
275 - *,
276 - default_sr: float,
277 - ) -> tuple[np.ndarray, float]:
278 - audio = self.audios[item_idx]
279 -
280 - if isinstance(audio, tuple):
281 - return audio
282 - if isinstance(audio, list):
283 - return np.array(audio), default_sr
284 - if isinstance(audio, np.ndarray):
285 - return audio, default_sr
286 -
287 - assert_never(audio)
288 -
289 - def resample_audios(self, new_sr: float, *, drop_sr: bool = True) -> None:
290 - """
291 - If :code:`drop_sr=True`, the audio items in this dictionary are updated
292 - to be NumPy arrays which implicitly means that their sampling rate is
293 - the same as the model's expected sampling rate; otherwise, they remain
294 - as :code:`(audio, new_sr)` tuples.
295 - """
296 - if not self.audios:
297 - return
298 -
299 - new_audios = []
300 - for item_idx in range(len(self.audios)):
301 - audio, sr = self.get_audio_with_sr(item_idx, default_sr=new_sr)
302 - audio = resample_audio(audio, orig_sr=sr, target_sr=new_sr)
303 -
304 - new_audios.append(audio if drop_sr else (audio, new_sr))
305 -
306 - self["audio"] = new_audios
307 -
308 -
203 class _TokenMatch(NamedTuple):
204 start_idx: int
205 end_idx: int
@@ -583,11 +477,124 @@ def iter_placeholders(
477 )
478
479
586 -class ProcessorInputs(NamedTuple):
587 - """Keyword arguments to :meth:`BaseMultiModalProcessor`"""
480 +@dataclass
481 +class ProcessorInputs:
482 + """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
483 prompt_text: str
484 mm_data: MultiModalDataDict
590 - mm_processor_kwargs: Mapping[str, object]
485 + hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
486 +
487 +
488 +class ProcessingCache:
489 +
490 + def __init__(self, capacity: int) -> None:
491 + super().__init__()
492 +
493 + # DEBUG: Set to None to disable
494 + self.debug_cache_hit_ratio_steps: Optional[int] = None
495 +
496 + self._cache = LRUCache[str, Mapping[str,
497 + MultiModalFieldItem]](capacity)
498 +
499 + def _maybe_log_cache_stats(self) -> None:
500 + steps = self.debug_cache_hit_ratio_steps
501 + if not steps:
502 + return
503 +
504 + cache_stats = self._cache.stat()
505 + if cache_stats.total % steps == 0:
506 + logger.debug("ProcessingCache: hit_ratio = %.2f",
507 + cache_stats.hit_ratio)
508 +
509 + def _serialize_item(self, obj: object) -> bytes:
510 + # Simple cases
511 + if isinstance(obj, str):
512 + return obj.encode("utf-8")
513 + if isinstance(obj, bytes):
514 + return obj
515 + if isinstance(obj, Image):
516 + return obj.tobytes()
517 +
518 + # Convertible to NumPy arrays
519 + if isinstance(obj, torch.Tensor):
520 + obj = obj.numpy()
521 + if isinstance(obj, (int, float)):
522 + obj = np.array(obj)
523 + if isinstance(obj, np.ndarray):
524 + return obj.tobytes()
525 +
526 + logger.warning(
527 + "No serialization method found for %s. "
528 + "Falling back to pickle.", type(obj))
529 +
530 + return pickle.dumps(obj)
531 +
532 + def _item_to_bytes(
533 + self,
534 + key: str,
535 + obj: object,
536 + ) -> Iterable[tuple[bytes, bytes]]:
537 + # Recursive cases
538 + if isinstance(obj, (list, tuple)):
539 + for i, elem in enumerate(obj):
540 + yield from self._item_to_bytes(f"{key}.{i}", elem)
541 + elif isinstance(obj, dict):
542 + for k, v in obj.items():
543 + yield from self._item_to_bytes(f"{key}.{k}", v)
544 + else:
545 + key_bytes = self._serialize_item(key)
546 + value_bytes = self._serialize_item(obj)
547 + yield key_bytes, value_bytes
548 +
549 + def _hash_kwargs(self, **kwargs: object) -> str:
550 + hasher = blake3()
551 +
552 + for k, v in kwargs.items():
553 + for k_bytes, v_bytes in self._item_to_bytes(k, v):
554 + hasher.update(k_bytes)
555 + hasher.update(v_bytes)
556 +
557 + return hasher.hexdigest()
558 +
559 + def get(
560 + self,
561 + model_id: str,
562 + modality: str,
563 + input_item: object,
564 + input_kwargs: Mapping[str, object],
565 + ) -> Optional[Mapping[str, MultiModalFieldItem]]:
566 + """
567 + Get a processed multi-modal item from the cache
568 + according to its dependencies, including:
569 +
570 + - The model ID
571 + - The modality of the item
572 + - The original data item passed to the HF processor
573 + - The configuration options of the HF processor
574 + """
575 + self._maybe_log_cache_stats()
576 +
577 + cache_key = self._hash_kwargs(model_id=model_id,
578 + **{modality: input_item},
579 + **input_kwargs)
580 + return self._cache.get(cache_key)
581 +
582 + def put(
583 + self,
584 + model_id: str,
585 + modality: str,
586 + input_item: object,
587 + input_kwargs: Mapping[str, object],
588 + output_kwargs: Mapping[str, MultiModalFieldItem],
589 + ) -> None:
590 + """
591 + Put a processed multi-modal item into the cache
592 + according to its dependencies (see :meth:`get`).
593 + """
594 + cache_key = self._hash_kwargs(model_id=model_id,
595 + **{modality: input_item},
596 + **input_kwargs)
597 + self._cache.put(cache_key, output_kwargs)
598
599
600 class BaseMultiModalProcessor(ABC):
@@ -595,18 +602,24 @@ class BaseMultiModalProcessor(ABC):
602 Abstract base class to process multi-modal inputs to be used in vLLM.
603 """
604
598 - def __init__(self, ctx: InputProcessingContext) -> None:
605 + def __init__(self,
606 + ctx: InputProcessingContext,
607 + *,
608 + cache: Optional[ProcessingCache] = None,
609 + enable_sanity_checks: bool = True) -> None:
610 super().__init__()
611
612 self.ctx = ctx
613 + self.cache = cache
614 + self.enable_sanity_checks = enable_sanity_checks
615
616 def __call__(
617 self,
618 prompt: str,
619 mm_data: MultiModalDataDict,
607 - mm_processor_kwargs: Mapping[str, object],
620 + hf_processor_mm_kwargs: Mapping[str, object],
621 ) -> MultiModalInputsV2:
609 - return self.apply(prompt, mm_data, mm_processor_kwargs)
622 + return self.apply(prompt, mm_data, hf_processor_mm_kwargs)
623
624 def _get_hf_processor(self) -> ProcessorMixin:
625 """
@@ -624,12 +637,21 @@ class BaseMultiModalProcessor(ABC):
637 ) -> MultiModalDataItems:
638 return MultiModalDataItems.from_dict(mm_data)
639
640 + @abstractmethod
641 + def _get_mm_fields_config(
642 + self,
643 + hf_inputs: BatchFeature,
644 + hf_processor_mm_kwargs: Mapping[str, object],
645 + ) -> Mapping[str, MultiModalFieldConfig]:
646 + """Given the HF-processed data, output the metadata of each field."""
647 + raise NotImplementedError
648 +
649 @abstractmethod
650 def _get_prompt_replacements(
651 self,
652 mm_items: MultiModalDataItems,
631 - hf_inputs: BatchFeature,
632 - mm_processor_kwargs: Mapping[str, object],
653 + hf_processor_mm_kwargs: Mapping[str, object],
654 + out_mm_kwargs: MultiModalKwargs,
655 ) -> list[PromptReplacement]:
656 """
657 Given the original multi-modal items for this modality
@@ -651,7 +673,7 @@ class BaseMultiModalProcessor(ABC):
673 return list(
674 iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
675
654 - def _get_processor_data(
676 + def _get_hf_mm_data(
677 self,
678 mm_items: MultiModalDataItems,
679 ) -> tuple[dict[str, Any], dict[str, Any]]:
@@ -669,7 +691,7 @@ class BaseMultiModalProcessor(ABC):
691 and v[0].ndim == 2):
692 # Pass through embedding inputs (multi)
693 passthrough_data[f"{k}_embeds"] = v
672 - else:
694 + elif len(v) > 0:
695 # Map keys to plural form, e.g.: image -> images
696 processor_data[f"{k}s"] = v
697 else:
@@ -679,39 +701,181 @@ class BaseMultiModalProcessor(ABC):
701
702 def _call_hf_processor(
703 self,
682 - hf_processor: ProcessorMixin,
704 prompt: str,
684 - processor_data: Mapping[str, object],
685 - mm_processor_kwargs: Mapping[str, object],
705 + # Not to be confused with `mm_data` in `self.apply`.
706 + # This refers to the data to be passed to HF processor.
707 + mm_data: Mapping[str, object],
708 + mm_kwargs: Mapping[str, object],
709 ) -> BatchFeature:
710 return self.ctx.call_hf_processor(
688 - hf_processor,
689 - prompt,
690 - processor_data,
691 - mm_processor_kwargs,
711 + self._get_hf_processor(**mm_kwargs),
712 + dict(text=prompt, **mm_data),
713 + mm_kwargs,
714 )
715
716 def _apply_hf_processor(
717 self,
696 - prompt: str,
718 + prompt_text: str,
719 mm_items: MultiModalDataItems,
698 - mm_processor_kwargs: Mapping[str, object],
699 - ) -> BatchFeature:
700 - # some mm_processor_kwargs may be used in processor initialization
701 - # instead of processor call
702 - hf_processor = self._get_hf_processor(**mm_processor_kwargs)
720 + hf_processor_mm_kwargs: Mapping[str, object],
721 + ) -> tuple[list[int], MultiModalKwargs]:
722 + """
723 + Apply the HF processor on the full prompt text and multi-modal data.
724 + """
725 + processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
726 +
727 + processed_data = self._call_hf_processor(
728 + prompt=prompt_text,
729 + mm_data=processor_data,
730 + mm_kwargs=hf_processor_mm_kwargs,
731 + )
732 + processed_data.update(passthrough_data)
733
704 - processor_data, passthrough_data = self._get_processor_data(mm_items)
734 + prompt_ids, = processed_data.pop("input_ids").tolist()
735
706 - hf_inputs = self._call_hf_processor(
707 - hf_processor,
708 - prompt=prompt,
709 - processor_data=processor_data,
710 - mm_processor_kwargs=mm_processor_kwargs,
736 + mm_kwargs = MultiModalKwargs.from_hf_inputs(
737 + processed_data,
738 + self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
739 + enable_sanity_checks=self.enable_sanity_checks,
740 )
712 - hf_inputs.update(passthrough_data)
741
714 - return hf_inputs
742 + return prompt_ids, mm_kwargs
743 +
744 + def _apply_hf_processor_missing(
745 + self,
746 + prompt_text: str,
747 + mm_missing_data_items: MultiModalDataItems,
748 + hf_processor_mm_kwargs: Mapping[str, object],
749 + ):
750 + """
751 + Apply the HF processor on the full prompt text, but only on the
752 + multi-modal data that are missing from the cache.
753 +
754 + Note: We pass prompt text and multi-modal data into the HF processor
755 + in separate calls to avoid HF prompt replacement being done for
756 + cached items; instead, we rely on our own prompt replacement logic
757 + for the full text.
758 + """
759 + mm_missing_counts = mm_missing_data_items.get_item_counts()
760 +
761 + prompt_ids, _ = self._apply_hf_processor(
762 + prompt_text=prompt_text,
763 + mm_items=MultiModalDataItems({}),
764 + hf_processor_mm_kwargs={},
765 + )
766 +
767 + # Some HF processors (e.g. Qwen2-VL) expect corresponding
768 + # multi-modal tokens to be in the prompt text
769 + dummy_inputs = self._get_dummy_mm_inputs(mm_missing_counts)
770 +
771 + _, mm_missing_kwargs = self._apply_hf_processor(
772 + prompt_text=dummy_inputs.prompt_text,
773 + mm_items=mm_missing_data_items,
774 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
775 + )
776 +
777 + return prompt_ids, mm_missing_kwargs
778 +
779 + def _cached_apply_hf_processor(
780 + self,
781 + prompt_text: str,
782 + mm_data_items: MultiModalDataItems,
783 + hf_processor_mm_kwargs: Mapping[str, object],
784 + ) -> tuple[list[int], MultiModalKwargs]:
785 + """
786 + Apply the HF processor on the full prompt text,
787 + caching the results and reusing cached results.
788 + """
789 + cache = self.cache
790 + model_id = self.ctx.model_config.model
791 +
792 + if cache is None or mm_data_items.has_embedding_inputs():
793 + return self._apply_hf_processor(
794 + prompt_text=prompt_text,
795 + mm_items=mm_data_items,
796 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
797 + )
798 +
799 + mm_maybe_cached_field_items = {
800 + modality: [
801 + cache.get(model_id, modality, item, hf_processor_mm_kwargs)
802 + for item in items
803 + ]
804 + for modality, items in mm_data_items.items()
805 + }
806 +
807 + mm_missing_idxs = {
808 + modality: [idx for idx, out in enumerate(fields) if out is None]
809 + for modality, fields in mm_maybe_cached_field_items.items()
810 + }
811 + mm_missing_data = {
812 + modality: [mm_data_items[modality][idx] for idx in idxs]
813 + for modality, idxs in mm_missing_idxs.items()
814 + }
815 + mm_missing_data_items = self._get_mm_items(mm_missing_data)
816 +
817 + prompt_ids, mm_missing_kwargs = self._apply_hf_processor_missing(
818 + prompt_text=prompt_text,
819 + mm_missing_data_items=mm_missing_data_items,
820 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
821 + )
822 +
823 + mm_missing_next_idx = {
824 + modality: 0
825 + for modality in mm_missing_data_items
826 + }
827 +
828 + mm_merged_field_items = dict[str, list[Mapping[str,
829 + MultiModalFieldItem]]]()
830 + for modality, modal_items_lst in mm_maybe_cached_field_items.items():
831 + merged_modal_items_lst = list[Mapping[str, MultiModalFieldItem]]()
832 +
833 + for idx, modal_items in enumerate(modal_items_lst):
834 + if modal_items is None:
835 + modal_items = mm_missing_kwargs.get_items_by_modality(
836 + modality,
837 + mm_missing_next_idx[modality],
838 + )
839 +
840 + cache.put(
841 + model_id,
842 + modality,
843 + mm_data_items[modality][idx],
844 + hf_processor_mm_kwargs,
845 + modal_items,
846 + )
847 +
848 + mm_missing_next_idx[modality] += 1
849 +
850 + merged_modal_items_lst.append(modal_items)
851 +
852 + mm_merged_field_items[modality] = merged_modal_items_lst
853 +
854 + if self.enable_sanity_checks:
855 + mm_missing_counts = mm_missing_data_items.get_item_counts()
856 + assert all(
857 + item_count == mm_missing_counts[modality]
858 + for modality, item_count in mm_missing_next_idx.items()), dict(
859 + mm_missing_next_idx=mm_missing_next_idx,
860 + mm_missing_counts=mm_missing_counts)
861 +
862 + mm_kwargs = MultiModalKwargs.from_items_by_modality(
863 + mm_merged_field_items,
864 + enable_sanity_checks=self.enable_sanity_checks,
865 + )
866 +
867 + if self.enable_sanity_checks:
868 + mm_item_counts = mm_data_items.get_item_counts()
869 +
870 + for modality, item_count in mm_item_counts.items():
871 + for item_idx in range(item_count):
872 + try:
873 + mm_kwargs.get_items_by_modality(modality, item_idx)
874 + except Exception as e:
875 + # Make it easy to set a breakpoint in the debugger
876 + raise e
877 +
878 + return prompt_ids, mm_kwargs
879
880 def _bind_prompt_replacements(
881 self,
@@ -730,6 +894,10 @@ class BaseMultiModalProcessor(ABC):
894 tokenizer = self._get_tokenizer()
895
896 token_matches = find_token_matches(token_ids, prompt_repls)
897 + mm_match_counts = {
898 + modality: len(matches)
899 + for modality, matches in full_groupby_modality(token_matches)
900 + }
901
902 # If the search text does not represent a special token,
903 # it may have different token IDs in the prompt, because
@@ -742,8 +910,8 @@ class BaseMultiModalProcessor(ABC):
910 # of the search text in the prompt, we instead perform string
911 # replacement on the decoded token IDs, then encode them back.
912 if all(
745 - len(matches) >= mm_item_counts[modality]
746 - for modality, matches in full_groupby_modality(token_matches)
913 + mm_match_counts.get(modality, 0) >= item_count
914 + for modality, item_count in mm_item_counts.items()
915 ): # yapf: disable
916 token_ids = replace_token_matches(
917 token_ids,
@@ -775,7 +943,7 @@ class BaseMultiModalProcessor(ABC):
943 self,
944 prompt_text: str,
945 mm_data: MultiModalDataDict,
778 - mm_processor_kwargs: Mapping[str, object],
946 + hf_processor_mm_kwargs: Mapping[str, object],
947 ) -> MultiModalInputsV2:
948 """
949 Process multi-modal inputs to be used in vLLM.
@@ -792,20 +960,24 @@ class BaseMultiModalProcessor(ABC):
960 """
961 mm_items = self._get_mm_items(mm_data)
962
795 - hf_inputs = self._apply_hf_processor(prompt_text, mm_items,
796 - mm_processor_kwargs)
797 - prompt_ids, = hf_inputs.pop("input_ids").tolist()
798 - mm_kwargs = MultiModalKwargs(hf_inputs)
963 + prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
964 + prompt_text,
965 + mm_items,
966 + hf_processor_mm_kwargs,
967 + )
968
800 - prompt_repls = self._get_prompt_replacements(mm_items, hf_inputs,
801 - mm_processor_kwargs)
802 - all_prompt_repls = self._bind_prompt_replacements(prompt_repls)
969 + unbound_prompt_repls = self._get_prompt_replacements(
970 + mm_items,
971 + hf_processor_mm_kwargs,
972 + mm_kwargs,
973 + )
974 + prompt_repls = self._bind_prompt_replacements(unbound_prompt_repls)
975
976 # If HF processor already inserts placeholder tokens,
977 # there is no need for us to insert them
978 mm_item_counts = mm_items.get_item_counts()
807 - all_placeholders = self._find_placeholders(all_prompt_repls,
808 - prompt_ids, mm_item_counts)
979 + all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
980 + mm_item_counts)
981
982 if all_placeholders:
983 tokenizer = self._get_tokenizer()
@@ -817,7 +989,7 @@ class BaseMultiModalProcessor(ABC):
989 all_placeholders,
990 ) = self._apply_prompt_replacements(
991 prompt_ids,
820 - all_prompt_repls,
992 + prompt_repls,
993 mm_item_counts,
994 )
995
@@ -855,23 +1027,29 @@ class BaseMultiModalProcessor(ABC):
1027 from vllm.sequence import SequenceData
1028
1029 processor_inputs = self._get_dummy_mm_inputs(mm_counts)
858 - mm_inputs = self.apply(*processor_inputs)
1030 + mm_inputs = self.apply(
1031 + prompt_text=processor_inputs.prompt_text,
1032 + mm_data=processor_inputs.mm_data,
1033 + hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1034 + )
1035
1036 prompt_token_ids = mm_inputs["prompt_token_ids"]
1037 placeholders_by_modality = mm_inputs["mm_placeholders"]
1038
863 - total_placeholders_by_modality = dict[str, int]()
864 - for modality, placeholders in placeholders_by_modality.items():
865 - num_placeholders = sum(item["length"] for item in placeholders)
866 - max_tokens = mm_max_tokens[modality]
867 -
868 - if num_placeholders != max_tokens:
869 - logger.warning(
870 - "The processed dummy data has a total of %d placeholder "
871 - "tokens for the '%s' modality, which is not the expected "
872 - "%d tokens.", num_placeholders, modality, max_tokens)
873 -
874 - total_placeholders_by_modality[modality] = num_placeholders
1039 + total_placeholders_by_modality = {
1040 + modality: sum(item["length"] for item in placeholders)
1041 + for modality, placeholders in placeholders_by_modality.items()
1042 + }
1043 + expected_placeholders_by_modality = {
1044 + modality: mm_max_tokens[modality]
1045 + for modality in placeholders_by_modality
1046 + }
1047 + if total_placeholders_by_modality != expected_placeholders_by_modality:
1048 + raise AssertionError(
1049 + f"The processed dummy data has a total of "
1050 + f"{total_placeholders_by_modality} placeholder tokens, which "
1051 + f"is not the expected {expected_placeholders_by_modality} "
1052 + "tokens.")
1053
1054 total_len = len(prompt_token_ids)
1055 if total_len > seq_len:
vllm/multimodal/registry.py
+30 -20
@@ -1,10 +1,9 @@
1 import functools
2 from collections import UserDict
3 -from typing import (TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional,
3 +from typing import (TYPE_CHECKING, Any, Dict, Mapping, Optional, Protocol,
4 Sequence, Type, TypeVar)
5
6 import torch.nn as nn
7 -from typing_extensions import TypeAlias
7
8 from vllm.inputs import InputProcessingContext
9 from vllm.logger import init_logger
@@ -15,7 +14,7 @@ from .audio import AudioPlugin
14 from .base import MultiModalInputMapper, MultiModalPlugin, MultiModalTokensCalc
15 from .image import ImagePlugin
16 from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
18 -from .processing import BaseMultiModalProcessor
17 +from .processing import BaseMultiModalProcessor, ProcessingCache
18 from .video import VideoPlugin
19
20 if TYPE_CHECKING:
@@ -23,15 +22,22 @@ if TYPE_CHECKING:
22
23 logger = init_logger(__name__)
24
25 +# TODO: Tune the MM cache size
26 +MM_CACHE_SIZE = 256
27 +
28 N = TypeVar("N", bound=Type[nn.Module])
29
28 -MultiModalProcessorFactory: TypeAlias = Callable[[InputProcessingContext],
29 - BaseMultiModalProcessor]
30 -"""
31 -Constructs a :class:`MultiModalProcessor` instance from the context.
30
33 -The processing metadata should be derived from the context.
34 -"""
31 +class MultiModalProcessorFactory(Protocol):
32 + """Constructs a :class:`MultiModalProcessor` instance from the context."""
33 +
34 + def __call__(
35 + self,
36 + ctx: InputProcessingContext,
37 + *,
38 + cache: Optional[ProcessingCache] = None,
39 + ) -> BaseMultiModalProcessor:
40 + ...
41
42
43 class _MultiModalLimits(UserDict["ModelConfig", Dict[str, int]]):
@@ -71,6 +77,8 @@ class MultiModalRegistry:
77
78 self._limits_by_model = _MultiModalLimits()
79
80 + self._processing_cache = ProcessingCache(MM_CACHE_SIZE)
81 +
82 def register_plugin(self, plugin: MultiModalPlugin) -> None:
83 """
84 Register a multi-modal plugin so it can be recognized by vLLM.
@@ -328,15 +336,18 @@ class MultiModalRegistry:
336
337 return wrapper
338
331 - def has_processor(self, model_config: "ModelConfig") -> bool:
332 - """
333 - Test whether a multi-modal processor is defined for a specific model.
334 - """
339 + def _get_model_cls(self, model_config: "ModelConfig"):
340 # Avoid circular import
341 from vllm.model_executor.model_loader import get_model_architecture
342
343 model_cls, _ = get_model_architecture(model_config)
339 - return model_cls in self._processor_factories
344 + return model_cls
345 +
346 + def has_processor(self, model_config: "ModelConfig") -> bool:
347 + """
348 + Test whether a multi-modal processor is defined for a specific model.
349 + """
350 + return self._get_model_cls(model_config) in self._processor_factories
351
352 def create_processor(
353 self,
@@ -346,12 +357,11 @@ class MultiModalRegistry:
357 """
358 Create a multi-modal processor for a specific model and tokenizer.
359 """
349 -
350 - # Avoid circular import
351 - from vllm.model_executor.model_loader import get_model_architecture
352 -
353 - model_cls, _ = get_model_architecture(model_config)
360 + model_cls = self._get_model_cls(model_config)
361 processor_factory = self._processor_factories[model_cls]
362
363 ctx = InputProcessingContext(model_config, tokenizer)
357 - return processor_factory(ctx)
364 + cache = (None if model_config.disable_mm_preprocessor_cache else
365 + self._processing_cache)
366 +
367 + return processor_factory(ctx, cache=cache)
vllm/transformers_utils/processor.py
+9 -3
@@ -1,25 +1,31 @@
1 from functools import lru_cache
2 from typing import Any, cast
3
4 +from transformers.processing_utils import ProcessorMixin
5 +
6
7 def get_processor(
8 processor_name: str,
9 *args: Any,
10 trust_remote_code: bool = False,
11 + processor_cls: type[ProcessorMixin] = ProcessorMixin,
12 **kwargs: Any,
13 ):
14 """Load a processor for the given model name via HuggingFace."""
15 # don't put this import at the top level
16 # it will call torch.cuda.device_count()
17 from transformers import AutoProcessor
15 - from transformers.processing_utils import ProcessorMixin
18 +
19 + processor_factory = (AutoProcessor
20 + if processor_cls == ProcessorMixin else processor_cls)
21
22 try:
18 - processor = AutoProcessor.from_pretrained(
23 + processor = processor_factory.from_pretrained(
24 processor_name,
25 *args,
26 trust_remote_code=trust_remote_code,
22 - **kwargs)
27 + **kwargs,
28 + )
29 except ValueError as e:
30 # If the error pertains to the processor class not existing or not
31 # currently being imported, suggest using the --trust-remote-code flag.
vllm/utils.py
+25 -2
@@ -25,11 +25,11 @@ import warnings
25 import weakref
26 from asyncio import FIRST_COMPLETED, AbstractEventLoop, Task
27 from collections import OrderedDict, UserDict, defaultdict
28 -from collections.abc import Iterable, Mapping
28 +from collections.abc import Hashable, Iterable, Mapping
29 from dataclasses import dataclass, field
30 from functools import lru_cache, partial, wraps
31 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
32 - Dict, Generator, Generic, Hashable, List, Literal,
32 + Dict, Generator, Generic, List, Literal, NamedTuple,
33 Optional, Tuple, Type, TypeVar, Union, overload)
34 from uuid import uuid4
35
@@ -194,13 +194,29 @@ class Counter:
194 self.counter = 0
195
196
197 +class CacheInfo(NamedTuple):
198 + hits: int
199 + total: int
200 +
201 + @property
202 + def hit_ratio(self) -> float:
203 + if self.total == 0:
204 + return 0
205 +
206 + return self.hits / self.total
207 +
208 +
209 class LRUCache(Generic[_K, _V]):
210 + """Note: This class is not thread safe!"""
211
212 def __init__(self, capacity: int) -> None:
213 self.cache = OrderedDict[_K, _V]()
214 self.pinned_items = set[_K]()
215 self.capacity = capacity
216
217 + self._hits = 0
218 + self._total = 0
219 +
220 def __contains__(self, key: _K) -> bool:
221 return key in self.cache
222
@@ -218,6 +234,9 @@ class LRUCache(Generic[_K, _V]):
234 def __delitem__(self, key: _K) -> None:
235 self.pop(key)
236
237 + def stat(self) -> CacheInfo:
238 + return CacheInfo(hits=self._hits, total=self._total)
239 +
240 def touch(self, key: _K) -> None:
241 self.cache.move_to_end(key)
242
@@ -226,8 +245,12 @@ class LRUCache(Generic[_K, _V]):
245 if key in self.cache:
246 value = self.cache[key]
247 self.cache.move_to_end(key)
248 +
249 + self._hits += 1
250 else:
251 value = default
252 +
253 + self._total += 1
254 return value
255
256 def put(self, key: _K, value: _V) -> None: