A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[V1][VLM] V1 support for selected single-image models. (#11632)
Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Isotr0py <2037008807@qq.com>
Roger Wang committed
Dec 31, 2024 at 13:17 UTC
e7c7c5e822a886e3dba202ca1b756c3260efffcc
19 files changed
+590
-636
docs/source/models/supported_models.md
+5
-5
@@ -570,28 +570,28 @@ See [this page](#generative-models) for more information on how to use generativ
570
- `rhymes-ai/Aria`
571
-
572
- ✅︎
573
- -
573
+ - ✅︎
574
* - `Blip2ForConditionalGeneration`
575
- BLIP-2
576
- T + I<sup>E</sup>
577
- `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc.
578
-
579
- ✅︎
580
- -
580
+ - ✅︎
581
* - `ChameleonForConditionalGeneration`
582
- Chameleon
583
- T + I
584
- `facebook/chameleon-7b` etc.
585
-
586
- ✅︎
587
- -
587
+ - ✅︎
588
* - `FuyuForCausalLM`
589
- Fuyu
590
- T + I
591
- `adept/fuyu-8b` etc.
592
-
593
- ✅︎
594
- -
594
+ - ✅︎
595
* - `ChatGLMModel`
596
- GLM-4V
597
- T + I
@@ -633,7 +633,7 @@ See [this page](#generative-models) for more information on how to use generativ
633
- `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
634
-
635
- ✅︎
636
- -
636
+ - ✅︎
637
* - `LlavaNextVideoForConditionalGeneration`
638
- LLaVA-NeXT-Video
639
- T + V
examples/offline_inference_vision_language.py
+8
-2
@@ -24,10 +24,13 @@ def run_aria(question: str, modality: str):
24
assert modality == "image"
25
model_name = "rhymes-ai/Aria"
26
27
+ # NOTE: Need L40 (or equivalent) to avoid OOM
28
llm = LLM(model=model_name,
29
tokenizer_mode="slow",
29
- trust_remote_code=True,
30
dtype="bfloat16",
31
+ max_model_len=4096,
32
+ max_num_seqs=2,
33
+ trust_remote_code=True,
34
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
35
36
prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
@@ -57,6 +60,7 @@ def run_chameleon(question: str, modality: str):
60
prompt = f"{question}<image>"
61
llm = LLM(model="facebook/chameleon-7b",
62
max_model_len=4096,
63
+ max_num_seqs=2,
64
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
65
stop_token_ids = None
66
return llm, prompt, stop_token_ids
@@ -257,7 +261,7 @@ def run_minicpmv(question: str, modality: str):
261
# 2.5
262
# model_name = "openbmb/MiniCPM-Llama3-V-2_5"
263
260
- #2.6
264
+ # 2.6
265
model_name = "openbmb/MiniCPM-V-2_6"
266
tokenizer = AutoTokenizer.from_pretrained(model_name,
267
trust_remote_code=True)
@@ -430,9 +434,11 @@ def run_pixtral_hf(question: str, modality: str):
434
435
model_name = "mistral-community/pixtral-12b"
436
437
+ # NOTE: Need L40 (or equivalent) to avoid OOM
438
llm = LLM(
439
model=model_name,
440
max_model_len=8192,
441
+ max_num_seqs=2,
442
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
443
)
444
tests/models/decoder_only/vision_language/test_models.py
+2
-5
@@ -140,10 +140,7 @@ VLM_TEST_SETTINGS = {
140
"aria": VLMTestInfo(
141
models=["rhymes-ai/Aria"],
142
tokenizer_mode="slow",
143
- test_type=(
144
- VLMTestType.IMAGE,
145
- VLMTestType.MULTI_IMAGE,
146
- ),
143
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
144
dtype="bfloat16",
145
prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
146
img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
@@ -179,6 +176,7 @@ VLM_TEST_SETTINGS = {
176
test_type=VLMTestType.IMAGE,
177
prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
178
max_model_len=4096,
179
+ max_num_seqs=2,
180
auto_cls=AutoModelForVision2Seq,
181
postprocess_inputs=model_utils.cast_dtype_post_processor(
182
"pixel_values"
@@ -201,7 +199,6 @@ VLM_TEST_SETTINGS = {
199
vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
200
num_logprobs=10,
201
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
204
- marks=[large_gpu_mark(min_gb=48)],
202
),
203
"glm4": VLMTestInfo(
204
models=["THUDM/glm-4v-9b"],
tests/multimodal/test_processing.py
+16
-13
@@ -528,7 +528,7 @@ def _rand_audio(
528
529
def _test_processing_cache_correctness(
530
model_id: str,
531
- modalities: set[str],
531
+ modalities: dict[str, bool],
532
hit_rate: float,
533
num_batches: int,
534
simplify_rate: float,
@@ -583,9 +583,8 @@ def _test_processing_cache_correctness(
583
partial(_rand_audio, rng, min_len=256, max_len=512, sr=16000),
584
}
585
input_max_count = {
586
- "image": 3,
587
- "video": 3,
588
- "audio": 3,
586
+ modality: 3 if supports_multi else 1
587
+ for modality, supports_multi in modalities.items()
588
}
589
590
for batch_idx in range(num_batches):
@@ -624,12 +623,16 @@ def _test_processing_cache_correctness(
623
624
# yapf: disable
625
@pytest.mark.parametrize(("model_id", "modalities"), [
627
- ("llava-hf/llava-1.5-7b-hf", {"image"}),
628
- ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image"}),
629
- ("mistral-community/pixtral-12b", {"image"}),
630
- ("Qwen/Qwen2-VL-2B-Instruct", {"image", "video"}),
631
- ("Qwen/Qwen2-Audio-7B-Instruct", {"audio"}),
632
- ("fixie-ai/ultravox-v0_3", {"audio"}),
626
+ ("rhymes-ai/Aria", {"image": True}),
627
+ ("Salesforce/blip2-opt-2.7b", {"image": False}),
628
+ ("facebook/chameleon-7b", {"image": True}),
629
+ ("adept/fuyu-8b", {"image": False}),
630
+ ("llava-hf/llava-1.5-7b-hf", {"image": True}),
631
+ ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
632
+ ("mistral-community/pixtral-12b", {"image": True}),
633
+ ("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
634
+ ("Qwen/Qwen2-Audio-7B-Instruct", {"audio": True}),
635
+ ("fixie-ai/ultravox-v0_3", {"audio": True}),
636
])
637
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
638
@pytest.mark.parametrize("num_batches", [32])
@@ -637,7 +640,7 @@ def _test_processing_cache_correctness(
640
# yapf: enable
641
def test_processing_cache_correctness(
642
model_id: str,
640
- modalities: set[str],
643
+ modalities: dict[str, bool],
644
hit_rate: float,
645
num_batches: int,
646
simplify_rate: float,
@@ -653,7 +656,7 @@ def test_processing_cache_correctness(
656
657
# yapf: disable
658
@pytest.mark.parametrize(("model_id", "modalities"), [
656
- ("microsoft/Phi-3-vision-128k-instruct", {"image"}),
659
+ ("microsoft/Phi-3-vision-128k-instruct", {"image": True}),
660
])
661
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
662
@pytest.mark.parametrize("num_batches", [32])
@@ -661,7 +664,7 @@ def test_processing_cache_correctness(
664
# yapf: enable
665
def test_processing_cache_correctness_phi3v(
666
model_id: str,
664
- modalities: set[str],
667
+ modalities: dict[str, bool],
668
hit_rate: float,
669
num_batches: int,
670
simplify_rate: float,
vllm/model_executor/models/aria.py
+80
-89
@@ -1,15 +1,15 @@
1
-import math
2
-from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
1
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
2
+ Union)
3
4
import torch
5
import torch.nn as nn
6
from torch.nn.init import trunc_normal_
7
-from transformers import LlamaConfig
7
+from transformers import BatchFeature, PretrainedConfig
8
9
from vllm.attention import AttentionMetadata
10
from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_rank
12
-from vllm.inputs import INPUT_REGISTRY, token_inputs
12
+from vllm.inputs import InputContext
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.fused_moe import FusedMoE
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -17,30 +17,27 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
from vllm.model_executor.layers.logits_processor import LogitsProcessor
18
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19
get_compressed_tensors_cache_scale)
20
-from vllm.model_executor.layers.sampler import (Sampler, SamplerOutput,
21
- SamplingMetadata)
20
+from vllm.model_executor.layers.sampler import (SamplerOutput,
21
+ SamplingMetadata, get_sampler)
22
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23
from vllm.model_executor.model_loader.weight_utils import (
24
default_weight_loader, maybe_remap_kv_scale_name)
25
-from vllm.model_executor.models.idefics2_vision_model import (
26
- Idefics2VisionTransformer)
27
-from vllm.model_executor.models.interfaces import SupportsMultiModal
28
-from vllm.model_executor.models.llama import (LlamaDecoderLayer, LlamaMLP,
29
- LlamaModel)
30
-from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper,
31
- is_pp_missing_parameter,
32
- maybe_prefix,
33
- merge_multimodal_embeddings)
25
from vllm.multimodal import MULTIMODAL_REGISTRY
35
-from vllm.multimodal.image import cached_get_image_processor
36
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
37
-from vllm.multimodal.utils import (cached_get_tokenizer,
38
- repeat_and_pad_placeholder_tokens)
26
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
27
+ NestedTensors)
28
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
+ MultiModalDataItems, ProcessorInputs,
30
+ PromptReplacement)
31
from vllm.sequence import IntermediateTensors
32
from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
33
AriaVisionConfig)
34
43
-from .utils import flatten_bn
35
+from .idefics2_vision_model import Idefics2VisionTransformer
36
+from .interfaces import SupportsMultiModal
37
+from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
38
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39
+ is_pp_missing_parameter, maybe_prefix,
40
+ merge_multimodal_embeddings)
41
42
43
class AriaImagePixelInputs(TypedDict):
@@ -251,7 +248,7 @@ class AriaProjector(nn.Module):
248
class AriaFusedMoE(FusedMoE):
249
250
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
254
- shard_id: str) -> Set[str]:
251
+ shard_id: str) -> None:
252
# Override the weight_loader to handle the expert weights in the Aria
253
# model, which are already packed with experts, and merge the gate and
254
# up weights for each expert.
@@ -346,7 +343,7 @@ class MoEDecoderLayer(LlamaDecoderLayer):
343
344
def __init__(
345
self,
349
- config: LlamaConfig,
346
+ config: AriaMoELMConfig,
347
cache_config: Optional[CacheConfig] = None,
348
quant_config: Optional[QuantizationConfig] = None,
349
prefix: str = "",
@@ -434,7 +431,7 @@ class AriaMoELMModel(LlamaModel):
431
return loaded_params
432
433
437
-def build_mm_projector(config):
434
+def build_mm_projector(config: PretrainedConfig):
435
return AriaProjector(
436
patch_to_query_dict=config.projector_patch_to_query_dict,
437
embed_dim=config.vision_config.hidden_size,
@@ -445,75 +442,70 @@ def build_mm_projector(config):
442
)
443
444
448
-def get_max_multimodal_tokens(ctx):
449
- return max(ctx.model_config.hf_config.image_size2tokens.values())
450
-
451
-
452
-def input_mapper_for_aria(ctx, data):
453
- return MultiModalKwargs(data)
445
+def get_max_aria_image_tokens(ctx: InputContext):
446
+ hf_config = ctx.get_hf_config()
447
+ return max(hf_config.projector_patch_to_query_dict.values())
448
449
456
-def input_processor(ctx, llm_inputs):
457
- multi_modal_data = llm_inputs.get("multi_modal_data")
458
- # if it is pure text input, use it as is
459
- if multi_modal_data is None or "image" not in multi_modal_data:
460
- return llm_inputs
450
+class AriaMultiModalProcessor(BaseMultiModalProcessor):
451
462
- model_config = ctx.model_config
463
-
464
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
465
- image_processor = cached_get_image_processor(
466
- model_config.model, trust_remote_code=model_config.trust_remote_code)
467
- hf_config = model_config.hf_config
452
+ def _get_mm_fields_config(
453
+ self,
454
+ hf_inputs: BatchFeature,
455
+ hf_processor_mm_kwargs: Mapping[str, object],
456
+ ) -> Mapping[str, MultiModalFieldConfig]:
457
+ return dict(
458
+ pixel_values=MultiModalFieldConfig.batched("image"),
459
+ pixel_mask=MultiModalFieldConfig.batched("image"),
460
+ )
461
469
- # prepare image tokens, the max_image_size is used to determine the number
470
- # of patch_size for every image
471
- max_image_size = multi_modal_data.pop("max_image_size", 980)
472
- _split_image = multi_modal_data.pop("split_image", False)
462
+ def _get_prompt_replacements(
463
+ self,
464
+ mm_items: MultiModalDataItems,
465
+ hf_processor_mm_kwargs: Mapping[str, object],
466
+ out_mm_kwargs: MultiModalKwargs,
467
+ ) -> list[PromptReplacement]:
468
+ hf_config = self.ctx.get_hf_config()
469
+ image_token_id = hf_config.image_token_index
470
+
471
+ max_image_tokens = get_max_aria_image_tokens(self.ctx)
472
+
473
+ return [
474
+ PromptReplacement(
475
+ modality="image",
476
+ target=[image_token_id],
477
+ replacement=[image_token_id] * max_image_tokens,
478
+ )
479
+ ]
480
474
- assert isinstance(max_image_size,
475
- (int, float)), "max_image_size should be float or int"
476
- images = (multi_modal_data["image"] if isinstance(
477
- multi_modal_data["image"], list) else [multi_modal_data["image"]])
481
+ def _get_dummy_mm_inputs(
482
+ self,
483
+ mm_counts: Mapping[str, int],
484
+ ) -> ProcessorInputs:
485
+ hf_config = self.ctx.get_hf_config()
486
+ vision_config: AriaVisionConfig = hf_config.vision_config
487
+
488
+ max_image_size = vision_config.image_size
489
+ num_images = mm_counts.get("image", 0)
490
+
491
+ mm_data = {
492
+ "image":
493
+ self._get_dummy_images(width=max_image_size,
494
+ height=max_image_size,
495
+ num_images=num_images)
496
+ }
497
479
- image_inputs = image_processor.preprocess(images,
480
- max_image_size=max_image_size,
481
- split_image=_split_image,
482
- return_tensors="pt").data
483
- image_inputs['pixel_values'] = image_inputs['pixel_values'].to(
484
- ctx.model_config.dtype)
485
- num_crops = image_inputs.pop("num_crops")
498
+ hf_processor = self._get_hf_processor()
499
+ image_token: str = hf_processor.image_token # type: ignore
500
487
- prompt_token_ids = llm_inputs["prompt_token_ids"]
488
- if num_crops.sum().item() > 0:
489
- _, prompt_token_ids, _ = repeat_and_pad_placeholder_tokens(
490
- tokenizer,
491
- None,
492
- prompt_token_ids,
493
- placeholder_token_id=hf_config.image_token_index,
494
- repeat_count=num_crops,
501
+ return ProcessorInputs(
502
+ prompt_text=image_token * num_images,
503
+ mm_data=mm_data,
504
)
505
497
- repeat_count = [hf_config.image_size2tokens[max_image_size]
498
- ] * sum(num_crops).item()
499
- new_prompt, new_token_ids, _ = repeat_and_pad_placeholder_tokens(
500
- tokenizer,
501
- None,
502
- prompt_token_ids,
503
- placeholder_token_id=hf_config.image_token_index,
504
- repeat_count=repeat_count,
505
- )
506
-
507
- return token_inputs(
508
- prompt_token_ids=new_token_ids,
509
- prompt=new_prompt,
510
- multi_modal_data={"image": image_inputs},
511
- )
512
-
506
514
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_multimodal_tokens)
515
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_aria)
516
-@INPUT_REGISTRY.register_input_processor(input_processor)
507
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_aria_image_tokens)
508
+@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
509
class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
510
"""
511
Aria model for conditional generation tasks.
@@ -540,12 +532,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
532
config = vllm_config.model_config.hf_config
533
quant_config = vllm_config.quant_config
534
543
- # prepare the image_size to tokens mapping for the image preprocess, see
544
- # input_processor
545
- config.image_size2tokens = {
546
- int(math.sqrt(k) * config.vision_config.patch_size): v
547
- for k, v in config.projector_patch_to_query_dict.items()
548
- }
535
self.config = config
536
self.vision_tower = AriaVisionModel(config.vision_config)
537
self.multi_modal_projector = build_mm_projector(config)
@@ -566,7 +552,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
552
logit_scale = getattr(config, "logit_scale", 1.0)
553
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
554
self.vocab_size, logit_scale)
569
- self.sampler = Sampler()
555
+ self.sampler = get_sampler()
556
557
def _validate_image_sizes(
558
self, images: List[torch.Tensor]) -> List[torch.Tensor]:
@@ -588,7 +574,12 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
574
575
pixel_values = self._validate_image_sizes(pixel_values)
576
pixel_values = flatten_bn(pixel_values, concat=True)
577
+
578
if pixel_mask is not None:
579
+ if not isinstance(pixel_mask, (torch.Tensor, list)):
580
+ raise ValueError("Incorrect type of pixel mask. "
581
+ f"Got type: {type(pixel_mask)}")
582
+
583
pixel_mask = flatten_bn(pixel_mask, concat=True)
584
585
return AriaImagePixelInputs(
vllm/model_executor/models/blip.py
-92
@@ -4,22 +4,16 @@ from typing import Iterable, Optional, Set, Tuple, Union
4
5
import torch
6
import torch.nn as nn
7
-from PIL import Image
7
from transformers import Blip2VisionConfig, BlipVisionConfig
8
9
from vllm.attention.layer import MultiHeadAttention
11
-from vllm.config import ModelConfig
10
from vllm.distributed import divide, get_tensor_model_parallel_world_size
13
-from vllm.inputs import DecoderOnlyInputs, token_inputs
11
from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
QKVParallelLinear,
14
RowParallelLinear)
15
from vllm.model_executor.layers.quantization import QuantizationConfig
16
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
-from vllm.multimodal.utils import (cached_get_tokenizer,
21
- repeat_and_pad_placeholder_tokens)
22
-from vllm.sequence import SequenceData
17
18
19
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -33,92 +27,6 @@ def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
27
return grid_length * grid_length
28
29
36
-def get_blip_image_feature_size(
37
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
38
- return get_blip_num_patches(image_size=hf_config.image_size,
39
- patch_size=hf_config.patch_size)
40
-
41
-
42
-def get_max_blip_image_tokens(
43
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
44
- return get_blip_image_feature_size(hf_config)
45
-
46
-
47
-def dummy_seq_data_for_blip(
48
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
49
- seq_len: int,
50
- num_images: int,
51
- *,
52
- image_token_id: int,
53
- image_feature_size_override: Optional[int] = None,
54
-):
55
- if image_feature_size_override is None:
56
- image_feature_size = get_blip_image_feature_size(hf_config)
57
- else:
58
- image_feature_size = image_feature_size_override
59
-
60
- return SequenceData.from_prompt_token_counts(
61
- (image_token_id, image_feature_size * num_images),
62
- (0, seq_len - image_feature_size * num_images),
63
- )
64
-
65
-
66
-def dummy_image_for_blip(
67
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
68
- num_images: int,
69
- *,
70
- image_width_override: Optional[int] = None,
71
- image_height_override: Optional[int] = None,
72
-):
73
- width = height = hf_config.image_size
74
- if image_width_override is not None:
75
- width = image_width_override
76
- if image_height_override is not None:
77
- height = image_height_override
78
-
79
- image = Image.new("RGB", (width, height), color=0)
80
- return {"image": image if num_images == 1 else [image] * num_images}
81
-
82
-
83
-def input_processor_for_blip(
84
- model_config: ModelConfig,
85
- hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
86
- inputs: DecoderOnlyInputs,
87
- *,
88
- image_token_id: int,
89
- image_feature_size_override: Optional[int] = None,
90
-):
91
- multi_modal_data = inputs.get("multi_modal_data")
92
- if multi_modal_data is None or "image" not in multi_modal_data:
93
- return inputs
94
-
95
- if "multi_modal_placeholders" in inputs and "image" in inputs[
96
- "multi_modal_placeholders"]:
97
- # The inputs already have placeholders.
98
- return inputs
99
-
100
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
101
-
102
- if image_feature_size_override is None:
103
- image_feature_size = get_blip_image_feature_size(hf_config)
104
- else:
105
- image_feature_size = image_feature_size_override
106
-
107
- new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
108
- tokenizer,
109
- inputs.get("prompt"),
110
- inputs["prompt_token_ids"],
111
- placeholder_token_id=image_token_id,
112
- repeat_count=image_feature_size,
113
- )
114
-
115
- # NOTE: Create a defensive copy of the original inputs
116
- return token_inputs(prompt_token_ids=new_token_ids,
117
- prompt=new_prompt,
118
- multi_modal_data=multi_modal_data,
119
- multi_modal_placeholders={"image": ranges})
120
-
121
-
30
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
31
class BlipVisionEmbeddings(nn.Module):
32
vllm/model_executor/models/blip2.py
+84
-88
@@ -4,32 +4,33 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
5
import torch
6
import torch.nn as nn
7
-from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
8
- apply_chunking_to_forward)
7
+from transformers import (BatchFeature, Blip2Config, Blip2Processor,
8
+ Blip2QFormerConfig, apply_chunking_to_forward)
9
10
from vllm.attention import AttentionMetadata
11
from vllm.config import CacheConfig, VllmConfig
12
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13
- InputContext, token_inputs)
12
+from vllm.inputs import InputContext
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
19
-from vllm.multimodal.inputs import NestedTensors
20
-from vllm.multimodal.utils import consecutive_placeholder_ranges
21
-from vllm.sequence import IntermediateTensors, SequenceData
22
-
23
-from .blip import (BlipVisionModel, dummy_image_for_blip,
24
- get_max_blip_image_tokens)
18
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
19
+ MultiModalInputsV2, MultiModalKwargs,
20
+ NestedTensors, PlaceholderRange)
21
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
22
+ MultiModalDataItems, ProcessorInputs,
23
+ PromptReplacement)
24
+from vllm.sequence import IntermediateTensors
25
+
26
+from .blip import BlipVisionModel
27
from .interfaces import SupportsMultiModal, SupportsPP
28
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
29
maybe_prefix, merge_multimodal_embeddings)
30
31
# We use this internally as placeholders since there is no image token
32
# defined on the HuggingFace repo
31
-BLIP2_IMAGE_TOKEN = "<image>"
32
-BLIP2_IMAGE_TOKEN_ID = 50265
33
+_IMAGE_TOKEN_ID = 50265
34
35
36
class Blip2ImagePixelInputs(TypedDict):
@@ -396,92 +397,87 @@ class Blip2QFormerModel(nn.Module):
397
return sequence_output
398
399
399
-def get_blip2_image_feature_size(hf_config: Blip2Config) -> int:
400
- return hf_config.num_query_tokens
401
-
402
-
400
def get_max_blip2_image_tokens(ctx: InputContext):
401
hf_config = ctx.get_hf_config(Blip2Config)
405
- vision_config = hf_config.vision_config
406
-
407
- if isinstance(vision_config, Blip2VisionConfig):
408
- return get_max_blip_image_tokens(vision_config)
409
-
410
- msg = f"Unsupported vision config: {type(vision_config)}"
411
- raise NotImplementedError(msg)
412
-
413
-
414
-def dummy_seq_data_for_blip2(
415
- hf_config: Blip2Config,
416
- seq_len: int,
417
- num_images: int,
418
- *,
419
- image_token_id: int,
420
- image_feature_size_override: Optional[int] = None,
421
-):
422
- if image_feature_size_override is None:
423
- image_feature_size = get_blip2_image_feature_size(hf_config)
424
- else:
425
- image_feature_size = image_feature_size_override
426
-
427
- return SequenceData.from_prompt_token_counts(
428
- (image_token_id, image_feature_size * num_images),
429
- (0, seq_len - image_feature_size * num_images),
430
- ), {
431
- "image":
432
- consecutive_placeholder_ranges(num_items=num_images,
433
- item_size=image_feature_size)
434
- }
435
-
436
-
437
-def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
438
- mm_counts: Mapping[str, int]):
439
- hf_config = ctx.get_hf_config(Blip2Config)
440
- vision_config = hf_config.vision_config
441
- num_images = mm_counts["image"]
442
-
443
- seq_data, ranges = dummy_seq_data_for_blip2(
444
- hf_config,
445
- seq_len,
446
- num_images,
447
- image_token_id=BLIP2_IMAGE_TOKEN_ID,
448
- )
449
-
450
- if isinstance(vision_config, Blip2VisionConfig):
451
- mm_data = dummy_image_for_blip(vision_config, num_images)
452
-
453
- return DummyData(seq_data, mm_data, ranges)
454
-
455
- msg = f"Unsupported vision config: {type(vision_config)}"
456
- raise NotImplementedError(msg)
402
+ return hf_config.num_query_tokens
403
404
459
-def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
460
- multi_modal_data = inputs.get("multi_modal_data")
461
- if multi_modal_data is None or "image" not in multi_modal_data:
462
- return inputs
405
+class Blip2MultiModalProcessor(BaseMultiModalProcessor):
406
464
- hf_config = ctx.get_hf_config(Blip2Config)
465
- image_feature_size = get_blip2_image_feature_size(hf_config)
407
+ def _get_hf_processor(self) -> Blip2Processor:
408
+ return self.ctx.get_hf_processor(Blip2Processor)
409
467
- # The original model places image tokens at the front
468
- # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
469
- new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
470
- new_token_ids += inputs["prompt_token_ids"]
410
+ def _get_mm_fields_config(
411
+ self,
412
+ hf_inputs: BatchFeature,
413
+ hf_processor_mm_kwargs: Mapping[str, object],
414
+ ) -> Mapping[str, MultiModalFieldConfig]:
415
+ return dict(
416
+ pixel_values=MultiModalFieldConfig.batched("image"),
417
+ image_embeds=MultiModalFieldConfig.batched("image"),
418
+ )
419
472
- new_prompt = inputs.get("prompt")
473
- if new_prompt is not None:
474
- new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
420
+ def _get_prompt_replacements(
421
+ self,
422
+ mm_items: MultiModalDataItems,
423
+ hf_processor_mm_kwargs: Mapping[str, object],
424
+ out_mm_kwargs: MultiModalKwargs,
425
+ ) -> list[PromptReplacement]:
426
+ max_image_tokens = get_max_blip2_image_tokens(self.ctx)
427
+
428
+ return [
429
+ PromptReplacement(
430
+ modality="image",
431
+ target="</s>",
432
+ replacement="<image>" * max_image_tokens + "</s>",
433
+ )
434
+ ]
435
476
- return token_inputs(prompt_token_ids=new_token_ids,
477
- prompt=new_prompt,
478
- multi_modal_data=multi_modal_data)
436
+ def apply(
437
+ self,
438
+ prompt_text: str,
439
+ mm_data: MultiModalDataDict,
440
+ hf_processor_mm_kwargs: Mapping[str, object],
441
+ ) -> MultiModalInputsV2:
442
+ result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
443
+
444
+ # Only <image> tokens should be considered as placeholders,
445
+ # so we ignore the trailing bos_token
446
+ result["mm_placeholders"] = {
447
+ modality: [
448
+ PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
449
+ for p in ps
450
+ ]
451
+ for modality, ps in result["mm_placeholders"].items()
452
+ }
453
+
454
+ return result
455
+
456
+ def _get_dummy_mm_inputs(
457
+ self,
458
+ mm_counts: Mapping[str, int],
459
+ ) -> ProcessorInputs:
460
+ hf_config = self.ctx.get_hf_config(Blip2Config)
461
+ vision_config = hf_config.vision_config
462
+
463
+ max_image_size = vision_config.image_size
464
+ num_images = mm_counts.get("image", 0)
465
+
466
+ mm_data = {
467
+ "image":
468
+ self._get_dummy_images(width=max_image_size,
469
+ height=max_image_size,
470
+ num_images=num_images)
471
+ }
472
+
473
+ return ProcessorInputs(
474
+ prompt_text="",
475
+ mm_data=mm_data,
476
+ )
477
478
481
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
479
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
483
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2)
484
-@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
480
+@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
481
class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
482
483
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -627,7 +623,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
623
if multimodal_embeddings is not None:
624
inputs_embeds = merge_multimodal_embeddings(
625
input_ids, inputs_embeds, multimodal_embeddings,
630
- BLIP2_IMAGE_TOKEN_ID)
626
+ _IMAGE_TOKEN_ID)
627
return inputs_embeds
628
629
def forward(
vllm/model_executor/models/chameleon.py
+81
-110
@@ -3,16 +3,15 @@ from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
3
Tuple, TypedDict, Union)
4
5
import torch
6
+import torch.nn as nn
7
import torch.nn.functional as F
7
-from PIL import Image
8
-from torch import nn
9
-from transformers import ChameleonConfig, ChameleonVQVAEConfig
8
+from transformers import (BatchFeature, ChameleonConfig, ChameleonProcessor,
9
+ ChameleonVQVAEConfig)
10
11
from vllm.attention import Attention, AttentionMetadata
12
from vllm.config import CacheConfig, VllmConfig
13
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
- InputContext, token_inputs)
14
+from vllm.inputs import InputContext
15
from vllm.model_executor.layers.activation import SiluAndMul
16
from vllm.model_executor.layers.layernorm import RMSNorm
17
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -29,11 +28,13 @@ from vllm.model_executor.model_loader.weight_utils import (
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
from vllm.model_executor.utils import set_weight_attrs
30
from vllm.multimodal import MULTIMODAL_REGISTRY
32
-from vllm.multimodal.inputs import NestedTensors
33
-from vllm.multimodal.utils import (cached_get_tokenizer,
34
- consecutive_placeholder_ranges,
35
- repeat_and_pad_placeholder_tokens)
36
-from vllm.sequence import IntermediateTensors, SequenceData
31
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32
+ MultiModalInputsV2, MultiModalKwargs,
33
+ NestedTensors, PlaceholderRange)
34
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
35
+ MultiModalDataItems, ProcessorInputs,
36
+ PromptReplacement)
37
+from vllm.sequence import IntermediateTensors
38
from vllm.utils import print_warning_once
39
40
from .interfaces import SupportsMultiModal, SupportsPP
@@ -45,10 +46,6 @@ from .utils import (is_pp_missing_parameter,
46
# and processor files, so we hardcode them in the model file for now.
47
CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
48
CHAMELEON_IMAGE_SEQ_LENGTH = 1024
48
-CHAMELEON_IMAGE_TOKEN_ID = 8711
49
-CHAMELEON_IMAGE_START_TOKEN_ID = 8197
50
-CHAMELEON_IMAGE_END_TOKEN_ID = 8196
51
-CHAMELEON_SEP_TOKEN_ID = 8710
49
50
51
class ChameleonImagePixelInputs(TypedDict):
@@ -61,99 +58,75 @@ def get_max_chameleon_image_tokens(ctx: InputContext):
58
return CHAMELEON_IMAGE_SEQ_LENGTH
59
60
64
-def dummy_seq_data_for_chameleon(
65
- seq_len: int,
66
- num_images: int,
67
- *,
68
- image_token_id: int,
69
- image_feature_size_override: Optional[int] = None,
70
-):
71
- if image_feature_size_override is None:
72
- image_feature_size = CHAMELEON_IMAGE_SEQ_LENGTH
73
- else:
74
- image_feature_size = image_feature_size_override
75
-
76
- return SequenceData.from_prompt_token_counts(
77
- (image_token_id, image_feature_size * num_images),
78
- (0, seq_len - image_feature_size * num_images),
79
- ), {
80
- "image":
81
- consecutive_placeholder_ranges(num_items=num_images,
82
- item_size=image_feature_size)
83
- }
84
-
85
-
86
-def dummy_image_for_chameleon(
87
- num_images: int,
88
- *,
89
- image_width_override: Optional[int] = None,
90
- image_height_override: Optional[int] = None,
91
-):
92
- width = CHAMELEON_CROP_SIZE_WIDTH
93
- height = CHAMELEON_CROP_SIZE_HEIGHT
94
- if image_width_override is not None:
95
- width = image_width_override
96
- if image_height_override is not None:
97
- height = image_height_override
98
-
99
- image = Image.new("RGB", (width, height), color=0)
100
- return {"image": image if num_images == 1 else [image] * num_images}
101
-
102
-
103
-def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
104
- mm_counts: Mapping[str, int]):
105
- num_images = mm_counts["image"]
106
-
107
- seq_data, ranges = dummy_seq_data_for_chameleon(
108
- seq_len,
109
- num_images,
110
- image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
111
- )
112
-
113
- mm_data = dummy_image_for_chameleon(num_images)
114
- return DummyData(seq_data, mm_data, ranges)
115
-
116
-
117
-def input_processor_for_chameleon(ctx: InputContext,
118
- inputs: DecoderOnlyInputs):
61
+class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
62
120
- """
121
- Processing input prompt to insert required tokens for image placeholder.
122
-
123
- See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58
124
- """ # noqa
125
-
126
- multi_modal_data = inputs.get("multi_modal_data")
127
- if multi_modal_data is None or "image" not in multi_modal_data:
128
- return inputs
129
-
130
- if "multi_modal_placeholders" in inputs and "image" in inputs[
131
- "multi_modal_placeholders"]:
132
- # The inputs already have placeholders.
133
- return inputs
134
-
135
- model_config = ctx.model_config
136
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
137
- new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
138
- tokenizer,
139
- inputs.get("prompt"),
140
- inputs["prompt_token_ids"],
141
- placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID,
142
- repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
143
- pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
144
- pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID,
145
- )
146
-
147
- # Appending sep token for chat mode to follow default processor
148
- # behavior
149
- if new_prompt is not None:
150
- new_prompt += tokenizer.sep_token
151
- new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
152
-
153
- # NOTE: Create a defensive copy of the original inputs
154
- return token_inputs(prompt_token_ids=new_token_ids,
155
- prompt=new_prompt,
156
- multi_modal_data=multi_modal_data)
63
+ def _get_hf_processor(self) -> ChameleonProcessor:
64
+ return self.ctx.get_hf_processor(ChameleonProcessor)
65
+
66
+ def _get_mm_fields_config(
67
+ self,
68
+ hf_inputs: BatchFeature,
69
+ hf_processor_mm_kwargs: Mapping[str, object],
70
+ ) -> Mapping[str, MultiModalFieldConfig]:
71
+ return dict(pixel_values=MultiModalFieldConfig.batched("image"))
72
+
73
+ def _get_prompt_replacements(
74
+ self,
75
+ mm_items: MultiModalDataItems,
76
+ hf_processor_mm_kwargs: Mapping[str, object],
77
+ out_mm_kwargs: MultiModalKwargs,
78
+ ) -> list[PromptReplacement]:
79
+ processor = self._get_hf_processor()
80
+
81
+ return [
82
+ PromptReplacement(
83
+ modality="image",
84
+ target="<image>",
85
+ replacement="".join([
86
+ processor.image_start_token,
87
+ processor.image_token * CHAMELEON_IMAGE_SEQ_LENGTH,
88
+ processor.image_end_token,
89
+ ]),
90
+ )
91
+ ]
92
+
93
+ def _get_dummy_mm_inputs(
94
+ self,
95
+ mm_counts: Mapping[str, int],
96
+ ) -> ProcessorInputs:
97
+ num_images = mm_counts.get("image", 0)
98
+
99
+ mm_data = {
100
+ "image":
101
+ self._get_dummy_images(width=CHAMELEON_CROP_SIZE_WIDTH,
102
+ height=CHAMELEON_CROP_SIZE_HEIGHT,
103
+ num_images=num_images)
104
+ }
105
+
106
+ return ProcessorInputs(
107
+ prompt_text="<image>" * num_images,
108
+ mm_data=mm_data,
109
+ )
110
+
111
+ def apply(
112
+ self,
113
+ prompt_text: str,
114
+ mm_data: MultiModalDataDict,
115
+ hf_processor_mm_kwargs: Mapping[str, object],
116
+ ) -> MultiModalInputsV2:
117
+ result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
118
+
119
+ # Only <image> tokens should be considered as placeholders,
120
+ # so we ignore the image_start_token and image_end_token
121
+ result["mm_placeholders"] = {
122
+ modality: [
123
+ PlaceholderRange(offset=p["offset"] + 1,
124
+ length=p["length"] - 2) for p in ps
125
+ ]
126
+ for modality, ps in result["mm_placeholders"].items()
127
+ }
128
+
129
+ return result
130
131
132
class ChameleonLayerNorm(nn.LayerNorm):
@@ -736,7 +709,7 @@ class ChameleonVQVAEEncoder(nn.Module):
709
for i_level in range(self.num_resolutions):
710
for i_block in range(self.num_res_blocks):
711
hidden_state = self.down[i_level].block[i_block](
739
- hidden_states[-1], )
712
+ hidden_states[-1])
713
if len(self.down[i_level].attn) > 0:
714
hidden_state = self.down[i_level].attn[i_block](
715
hidden_state)
@@ -925,10 +898,8 @@ class ChameleonModel(nn.Module):
898
return hidden_states
899
900
928
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
901
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
930
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon)
931
-@INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon)
902
+@MULTIMODAL_REGISTRY.register_processor(ChameleonMultiModalProcessor)
903
class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
904
SupportsPP):
905
vllm/model_executor/models/fuyu.py
+199
-182
@@ -15,32 +15,30 @@
15
# limitations under the License.
16
""" PyTorch Fuyu model."""
17
import math
18
-from array import array
18
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
19
TypedDict)
20
21
import torch
22
import torch.nn as nn
24
-import torch.utils.checkpoint
25
-from PIL import Image
26
-from transformers import FuyuImageProcessor
23
+from transformers import (BatchFeature, FuyuConfig, FuyuImageProcessor,
24
+ FuyuProcessor)
25
26
from vllm.attention import AttentionMetadata
27
from vllm.config import VllmConfig
30
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
31
- InputContext, token_inputs)
28
+from vllm.inputs import InputContext
29
from vllm.model_executor.layers.linear import ColumnParallelLinear
30
from vllm.model_executor.layers.sampler import SamplerOutput
31
from vllm.model_executor.models.persimmon import PersimmonForCausalLM
32
from vllm.model_executor.sampling_metadata import SamplingMetadata
36
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
37
-from vllm.multimodal.image import cached_get_image_processor
38
-from vllm.multimodal.inputs import NestedTensors
39
-from vllm.multimodal.utils import (cached_get_tokenizer,
40
- consecutive_placeholder_ranges)
41
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
- SequenceData)
43
-from vllm.utils import is_list_of
33
+from vllm.multimodal import MULTIMODAL_REGISTRY
34
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
+ MultiModalInputsV2, MultiModalKwargs,
36
+ NestedTensors, PlaceholderRange)
37
+from vllm.multimodal.parse import ImageProcessorItems
38
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
+ MultiModalDataItems, ProcessorInputs,
40
+ PromptReplacement)
41
+from vllm.sequence import IntermediateTensors
42
43
from .interfaces import SupportsMultiModal, SupportsPP
44
from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
@@ -54,178 +52,193 @@ MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080
52
MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
53
54
57
-class FuyuImagePixelInputs(TypedDict):
58
- type: Literal["pixel_values"]
55
+class FuyuImagePatchInputs(TypedDict):
56
+ type: Literal["image_patches"]
57
data: torch.Tensor
58
"""
59
Shape:
62
- (batch_size, num_patches, patch_size_x * patch_size_y * num_channels)
60
+ `(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`
61
+ """
62
+
63
+ patches_per_image: List[int]
64
+ """
65
+ List of number of total patches for each image in the batch.
66
+ This is used to restore the first two dimensions of `data`.
67
"""
68
69
66
-def _calculate_num_image_tokens(
67
- height: int,
68
- width: int,
70
+def _get_fuyu_num_image_tokens(
71
+ image_height: int,
72
+ image_width: int,
73
) -> Tuple[int, int]:
74
"""
71
- calculate number of image tokens needed for a given image size
72
- The expected Fuyu image prompts is in format:
73
- (image_token * ncols + newline_token) * nrows
74
- args:
75
- image_size: Tuple[int, int] - (width, height) of the image
76
- returns:
77
- ncols: int - number of image tokens in x direction
78
- nrows: int - number of image tokens in y direction
79
- """
80
- ncol = math.ceil(width / 30)
81
- nrow = math.ceil(height / 30)
82
- return ncol, nrow
75
+ Calculate the number of image tokens needed for a given image size.
76
77
+ The expected Fuyu image prompts can be expressed as:
78
85
-def get_max_fuyu_image_feature_size():
79
+ .. code-block::
80
+ (image_token * ncols + newline_token) * nrows
81
87
- return _calculate_num_image_tokens(
88
- height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
89
- width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
90
- )
82
+ Args:
83
+ image_size: Tuple[int, int] - `(width, height)` of the image
84
+
85
+ Returns:
86
+ ncols: int - number of image tokens in `x` direction
87
+ nrows: int - number of image tokens in `y` direction
88
+ """
89
+ ncols = math.ceil(image_width / 30)
90
+ nrows = math.ceil(image_height / 30)
91
+ return ncols, nrows
92
93
94
def get_max_fuyu_image_tokens(ctx: InputContext):
94
- ncol, nrow = get_max_fuyu_image_feature_size()
95
- return (ncol + 1) * nrow
96
-
97
-
98
-def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
99
- ncol, nrow = get_max_fuyu_image_feature_size()
100
- image_feature_size = get_max_fuyu_image_tokens(ctx)
101
-
102
- image_token_ids = (
103
- array(VLLM_TOKEN_ID_ARRAY_TYPE, [_IMAGE_TOKEN_ID]) * ncol +
104
- array(VLLM_TOKEN_ID_ARRAY_TYPE, [_NEWLINE_TOKEN_ID])) * nrow
105
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images
106
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
107
- [0]) * (seq_len - image_feature_size * num_images)
108
- return SequenceData(token_ids), {
109
- "image":
110
- consecutive_placeholder_ranges(num_items=num_images,
111
- item_size=image_feature_size)
112
- }
113
-
114
-
115
-def dummy_image_for_fuyu(
116
- num_images: int,
117
- *,
118
- image_width: int,
119
- image_height: int,
120
-):
121
- image = Image.new("RGB", (image_width, image_height), color=0)
122
- return {"image": image if num_images == 1 else [image] * num_images}
123
-
124
-
125
-def dummy_data_for_fuyu(ctx: InputContext, seq_len: int,
126
- mm_counts: Mapping[str, int]):
127
- num_images = mm_counts["image"]
128
- seq_data, ranges = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
129
- mm_data = dummy_image_for_fuyu(num_images,
130
- image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
131
- image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT)
132
- return DummyData(seq_data, mm_data, ranges)
133
-
134
-
135
-def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
136
- data: List[Image.Image]):
137
- image_encoding = image_processor.preprocess(data, return_tensors="pt")
138
- batch_images = torch.stack([img[0] for img in image_encoding["images"]
139
- ]).unsqueeze(1)
140
- image_unpadded_heights = torch.tensor(
141
- image_encoding["image_unpadded_heights"])
142
- image_unpadded_widths = torch.tensor(
143
- image_encoding["image_unpadded_widths"])
144
-
145
- batch_size = len(image_encoding["images"])
146
- image_present = torch.ones(batch_size, 1, 1)
147
- model_image_input = image_processor.preprocess_with_tokenizer_info(
148
- image_input=batch_images,
149
- image_present=image_present,
150
- image_unpadded_h=image_unpadded_heights,
151
- image_unpadded_w=image_unpadded_widths,
152
- image_placeholder_id=_IMAGE_TOKEN_ID,
153
- image_newline_id=_NEWLINE_TOKEN_ID,
154
- variable_sized=True,
95
+ ncols, nrows = _get_fuyu_num_image_tokens(
96
+ image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
97
+ image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
98
)
156
- return model_image_input
157
-
158
-
159
-def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
160
- multi_modal_data = inputs.get("multi_modal_data")
161
- if multi_modal_data is None or "image" not in multi_modal_data:
162
- return inputs
163
-
164
- model_config = ctx.model_config
165
- image_data = multi_modal_data["image"]
166
- new_multi_modal_data = {}
167
- image_list = image_data if isinstance(image_data, list) else [image_data]
168
-
169
- # process image data
170
- if is_list_of(image_list, Image.Image):
171
- # Fuyu's image_processor can also finish token padding
172
- image_processor: FuyuImageProcessor = cached_get_image_processor(
173
- model_config.model)
174
-
175
- model_image_input = _fuyu_image_preprocess(image_processor, image_data)
176
- image_patches = torch.cat([
177
- image_patch[0]
178
- for image_patch in model_image_input["image_patches"]
179
- ])
180
- new_multi_modal_data["image"] = image_patches
181
-
182
- elif is_list_of(image_list, torch.Tensor):
183
- raise NotImplementedError("Embeddings input is not supported yet")
184
- else:
185
- raise TypeError(f"Invalid image type: {type(image_data)}")
186
-
187
- # process prompts
188
- prompt = inputs.get("prompt")
189
- prompt_token_ids = inputs["prompt_token_ids"]
190
- tokenizer = cached_get_tokenizer(model_config.model)
191
- # dim0 is batch_size, dim1 is subseq_size which will always be 1
192
- image_input_ids: List[List[
193
- torch.Tensor]] = model_image_input["image_input_ids"]
194
- image_input_ids = image_input_ids[0][0].tolist()
195
- bos_token = tokenizer.encode("<s>", add_special_tokens=False)[1:]
196
- boa_token = tokenizer.encode("\x04", add_special_tokens=False)[1:]
197
-
198
- new_prompt = prompt + "\x04"
199
- new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[
200
- 1:] + boa_token
201
-
202
- return token_inputs(prompt=new_prompt,
203
- prompt_token_ids=new_prompt_token_ids,
204
- multi_modal_data=new_multi_modal_data)
205
-
206
-
207
-def input_mapper_for_fuyu(ctx: InputContext, data: object):
208
- model_config = ctx.model_config
209
- data_list = data if isinstance(data, list) else [data]
210
- if is_list_of(data_list, Image.Image):
211
- # Fuyu's image_processor can also finish token padding
212
- image_processor: FuyuImageProcessor = cached_get_image_processor(
213
- model_config.model)
214
-
215
- model_image_input = _fuyu_image_preprocess(image_processor, data_list)
216
- data = torch.stack([
217
- image_patch[0]
218
- for image_patch in model_image_input["image_patches"]
219
- ])
220
-
221
- # image has been processed with prompt in input processor
222
- return MultiModalKwargs({"pixel_values": data})
223
-
224
-
225
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu)
99
+
100
+ return (ncols + 1) * nrows
101
+
102
+
103
+class FuyuMultiModalProcessor(BaseMultiModalProcessor):
104
+
105
+ def _get_hf_processor(self) -> FuyuProcessor:
106
+ return self.ctx.get_hf_processor(FuyuProcessor)
107
+
108
+ def _call_hf_processor(
109
+ self,
110
+ prompt: str,
111
+ mm_data: Mapping[str, object],
112
+ mm_kwargs: Mapping[str, object],
113
+ ) -> BatchFeature:
114
+
115
+ if not mm_data:
116
+ # Avoid warning from HF logger for text-only input
117
+ # Input_ids format: bos_token_id + prompt_token_ids + boa_token_id
118
+ # Tokenizer won't add boa_token_id by default, we add it manually.
119
+ tokenizer = self._get_tokenizer()
120
+ boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
121
+ prompt_ids = tokenizer.encode(prompt) + [boa_token_id]
122
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
123
+
124
+ processed_outputs = super()._call_hf_processor(
125
+ prompt=prompt,
126
+ mm_data=mm_data,
127
+ mm_kwargs=mm_kwargs,
128
+ )
129
+
130
+ image_patches = processed_outputs.get("image_patches")
131
+ if image_patches is not None:
132
+ images = mm_data["images"]
133
+ assert isinstance(images, list)
134
+
135
+ # Original output: (1, num_images, Pn, Px * Py * C)
136
+ # New output: (num_images, Pn, Px * Py * C)
137
+ assert (isinstance(image_patches, list)
138
+ and len(image_patches) == 1)
139
+ assert (isinstance(image_patches[0], torch.Tensor)
140
+ and len(image_patches[0]) == len(images))
141
+
142
+ processed_outputs["image_patches"] = image_patches[0]
143
+
144
+ return processed_outputs
145
+
146
+ def _get_mm_fields_config(
147
+ self,
148
+ hf_inputs: BatchFeature,
149
+ hf_processor_mm_kwargs: Mapping[str, object],
150
+ ) -> Mapping[str, MultiModalFieldConfig]:
151
+ return dict(image_patches=MultiModalFieldConfig.batched("image"))
152
+
153
+ def _get_prompt_replacements(
154
+ self,
155
+ mm_items: MultiModalDataItems,
156
+ hf_processor_mm_kwargs: Mapping[str, object],
157
+ out_mm_kwargs: MultiModalKwargs,
158
+ ) -> list[PromptReplacement]:
159
+ hf_config = self.ctx.get_hf_config(FuyuConfig)
160
+ bos_token_id = hf_config.bos_token_id
161
+
162
+ tokenizer = self._get_tokenizer()
163
+ eot_token_id = tokenizer.bos_token_id
164
+ assert isinstance(eot_token_id, int)
165
+
166
+ hf_processor = self._get_hf_processor()
167
+ image_processor: FuyuImageProcessor = hf_processor.image_processor
168
+ target_size = image_processor.size
169
+ target_height, target_width = (target_size["height"],
170
+ target_size["width"])
171
+
172
+ def get_replacement_fuyu(item_idx: int):
173
+ images = mm_items.get_items("image", ImageProcessorItems)
174
+ image_size = images.get_image_size(item_idx)
175
+ width, height = image_size.width, image_size.height
176
+ if not (width <= target_width and height <= target_height):
177
+ height_scale_factor = target_height / height
178
+ width_scale_factor = target_width / width
179
+ optimal_scale_factor = min(height_scale_factor,
180
+ width_scale_factor)
181
+
182
+ height = int(height * optimal_scale_factor)
183
+ width = int(width * optimal_scale_factor)
184
+
185
+ ncols, nrows = _get_fuyu_num_image_tokens(
186
+ image_width=width,
187
+ image_height=height,
188
+ )
189
+
190
+ return (([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows +
191
+ [bos_token_id])
192
+
193
+ return [
194
+ PromptReplacement(
195
+ modality="image",
196
+ target=[eot_token_id],
197
+ replacement=get_replacement_fuyu,
198
+ )
199
+ ]
200
+
201
+ def apply(
202
+ self,
203
+ prompt_text: str,
204
+ mm_data: MultiModalDataDict,
205
+ hf_processor_mm_kwargs: Mapping[str, object],
206
+ ) -> MultiModalInputsV2:
207
+ result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
208
+
209
+ # Only |SPEAKER| (image) tokens should be considered as placeholders,
210
+ # so we ignore the trailing bos_token_id
211
+ result["mm_placeholders"] = {
212
+ modality: [
213
+ PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
214
+ for p in ps
215
+ ]
216
+ for modality, ps in result["mm_placeholders"].items()
217
+ }
218
+
219
+ return result
220
+
221
+ def _get_dummy_mm_inputs(
222
+ self,
223
+ mm_counts: Mapping[str, int],
224
+ ) -> ProcessorInputs:
225
+ num_images = mm_counts.get("image", 0)
226
+
227
+ mm_data = {
228
+ "image":
229
+ self._get_dummy_images(width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
230
+ height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
231
+ num_images=num_images)
232
+ }
233
+
234
+ return ProcessorInputs(
235
+ prompt_text="",
236
+ mm_data=mm_data,
237
+ )
238
+
239
+
240
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
227
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu)
228
-@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
241
+@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
242
class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
243
244
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -280,28 +293,32 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
293
return data.to(self.vision_embed_tokens.weight.dtype)
294
295
def _parse_and_validate_image_input(
283
- self, **kwargs: object) -> Optional[FuyuImagePixelInputs]:
284
- pixel_values = kwargs.pop("pixel_values", None)
285
-
286
- if pixel_values is not None:
287
- if not isinstance(pixel_values, (torch.Tensor, list)):
296
+ self, **kwargs: object) -> Optional[FuyuImagePatchInputs]:
297
+ image_patches = kwargs.pop("image_patches", None)
298
+ if image_patches is not None:
299
+ if not isinstance(image_patches, (torch.Tensor, list)):
300
raise ValueError("Incorrect type of image patches. "
289
- f"Got type: {type(pixel_values)}")
301
+ f"Got type: {type(image_patches)}")
302
291
- return FuyuImagePixelInputs(
292
- type="pixel_values",
303
+ image_patches_flat = flatten_bn(image_patches)
304
+
305
+ return FuyuImagePatchInputs(
306
+ type="image_patches",
307
data=self._validate_pixel_values(
294
- flatten_bn(pixel_values, concat=True)),
308
+ flatten_bn(image_patches_flat, concat=True)),
309
+ patches_per_image=[x.size(0) for x in image_patches_flat],
310
)
311
312
return None
313
314
def _process_image_input(
300
- self, image_input: FuyuImagePixelInputs) -> torch.Tensor:
315
+ self, image_input: FuyuImagePatchInputs) -> NestedTensors:
316
+ image_patches = image_input["data"]
317
+ patches_per_image = image_input["patches_per_image"]
318
319
assert self.vision_embed_tokens is not None
303
- vision_embeddings, _ = self.vision_embed_tokens(image_input["data"])
304
- return vision_embeddings
320
+ vision_embeddings, _ = self.vision_embed_tokens(image_patches)
321
+ return vision_embeddings.split(patches_per_image, dim=0)
322
323
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
324
image_input = self._parse_and_validate_image_input(**kwargs)
vllm/model_executor/models/idefics2_vision_model.py
+4
-2
@@ -69,7 +69,8 @@ class Idefics2VisionEmbeddings(nn.Module):
69
patch_attention_mask: torch.BoolTensor,
70
tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
71
batch_size, _, max_im_h, max_im_w = pixel_values.shape
72
- patch_embeds = self.patch_embedding(pixel_values)
72
+ target_dtype = self.patch_embedding.weight.dtype
73
+ patch_embeds = self.patch_embedding(pixel_values.to(target_dtype))
74
embeddings = patch_embeds.flatten(2).transpose(1, 2)
75
max_nb_patches_h, max_nb_patches_w = (
76
max_im_h // self.patch_size,
@@ -309,7 +310,8 @@ class Idefics2VisionTransformer(nn.Module):
310
hidden_states = self.embeddings(
311
pixel_values=pixel_values,
312
patch_attention_mask=patch_attention_mask,
312
- tgt_sizes=tgt_sizes)
313
+ tgt_sizes=tgt_sizes,
314
+ )
315
encoder_outputs = self.encoder(hidden_states)
316
last_hidden_state = self.post_layernorm(encoder_outputs)
317
return last_hidden_state
vllm/model_executor/models/llava.py
+2
-2
@@ -144,8 +144,8 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
144
# Original output: (1, num_images, C, H, W)
145
# New output: (num_images, C, H, W)
146
assert (isinstance(pixel_values, list)
147
- and len(pixel_values) == 1
148
- and isinstance(pixel_values[0], list)
147
+ and len(pixel_values) == 1)
148
+ assert (isinstance(pixel_values[0], list)
149
and len(pixel_values[0]) == len(images))
150
151
processed_outputs["pixel_values"] = pixel_values[0]
vllm/model_executor/models/llava_next.py
+2
-4
@@ -528,10 +528,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
528
stacked_image_features = self._image_pixels_to_features(
529
self.vision_tower, stacked_pixel_values)
530
531
- return [
532
- self.multi_modal_projector(image_features) for image_features in
533
- torch.split(stacked_image_features, num_patches_per_batch)
534
- ]
531
+ return torch.split(self.multi_modal_projector(stacked_image_features),
532
+ num_patches_per_batch)
533
534
def _process_image_input(
535
self,
vllm/model_executor/models/pixtral.py
+6
-6
@@ -1,8 +1,8 @@
1
+import math
2
from dataclasses import dataclass, fields
3
from functools import cached_property
4
from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union
5
5
-import numpy
6
import torch
7
import torch.nn as nn
8
import torch.nn.functional as F
@@ -306,7 +306,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
306
images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor],
307
torch.Tensor]] = None,
308
image_tokens: Optional[torch.Tensor] = None,
309
- ) -> Optional[List[torch.Tensor]]:
309
+ ) -> Tuple[Optional[List[torch.Tensor]], Optional[torch.Tensor]]:
310
if images is None:
311
return None, None
312
@@ -604,11 +604,11 @@ class VisionTransformer(nn.Module):
604
return self.args.image_size // self.args.patch_size
605
606
@property
607
- def device(self) -> torch.device:
607
+ def device(self) -> torch.types.Device:
608
return next(self.parameters()).device
609
610
@property
611
- def dtype(self) -> torch.device:
611
+ def dtype(self) -> torch.dtype:
612
return next(self.parameters()).dtype
613
614
@property
@@ -741,8 +741,8 @@ def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
741
ratio = max(image_width / max_width, image_height / max_height)
742
743
if ratio > 1:
744
- image_width = int(numpy.ceil(image_width / ratio))
745
- image_height = int(numpy.ceil(image_height / ratio))
744
+ image_width = int(math.ceil(image_width / ratio))
745
+ image_height = int(math.ceil(image_height / ratio))
746
747
num_height_tokens, num_width_tokens = _get_pixtral_hf_num_image_tokens(
748
(image_height, image_width),
vllm/model_executor/models/qwen2_audio.py
+8
-6
@@ -23,7 +23,6 @@ from functools import cached_property
23
from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24
Union)
25
26
-import numpy as np
26
import torch
27
import torch.nn as nn
28
from transformers import BatchFeature
@@ -177,16 +176,19 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
176
mm_counts: Mapping[str, int],
177
) -> ProcessorInputs:
178
feature_extractor = self._get_feature_extractor()
179
+
180
sampling_rate = feature_extractor.sampling_rate
181
audio_len = feature_extractor.chunk_length * sampling_rate
182
+ num_audios = mm_counts.get("audio", 0)
183
183
- audio_count = mm_counts.get("audio", 0)
184
- audio = np.zeros(audio_len)
185
- data = {"audio": [audio] * audio_count}
184
+ mm_data = {
185
+ "audio":
186
+ self._get_dummy_audios(length=audio_len, num_audios=num_audios)
187
+ }
188
189
return ProcessorInputs(
188
- prompt_text="<|AUDIO|>" * audio_count,
189
- mm_data=data,
190
+ prompt_text="<|AUDIO|>" * num_audios,
191
+ mm_data=mm_data,
192
)
193
194
vllm/model_executor/models/qwen2_vl.py
+9
-8
@@ -29,7 +29,6 @@ import torch
29
import torch.nn as nn
30
import torch.nn.functional as F
31
from einops import rearrange, repeat
32
-from PIL import Image
32
from transformers import BatchFeature
33
from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
34
Qwen2VLProcessor)
@@ -882,12 +881,10 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
881
self,
882
mm_counts: Mapping[str, int],
883
) -> ProcessorInputs:
885
- num_images = mm_counts.get("image", 0)
884
hf_processor = self._get_hf_processor()
887
- image_token: str = hf_processor.image_token
885
image_processor = _get_image_processor(hf_processor)
886
890
- data = {}
887
+ image_token: str = hf_processor.image_token
888
resized_height, resized_width = smart_resize(
889
height=9999999,
890
width=9999999,
@@ -895,14 +892,18 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
892
min_pixels=image_processor.min_pixels,
893
max_pixels=image_processor.max_pixels,
894
)
895
+ num_images = mm_counts.get("image", 0)
896
899
- dummy_image = Image.new("RGB", (resized_width, resized_height),
900
- color=0)
901
- data["image"] = [dummy_image] * num_images
897
+ mm_data = {
898
+ "image":
899
+ self._get_dummy_images(width=resized_width,
900
+ height=resized_height,
901
+ num_images=num_images)
902
+ }
903
904
return ProcessorInputs(
905
prompt_text=image_token * num_images,
905
- mm_data=data,
906
+ mm_data=mm_data,
907
)
908
909
vllm/model_executor/models/ultravox.py
+8
-5
@@ -188,16 +188,19 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
188
mm_counts: Mapping[str, int],
189
) -> ProcessorInputs:
190
feature_extractor = self._get_feature_extractor()
191
+
192
sampling_rate = feature_extractor.sampling_rate
193
audio_len = feature_extractor.chunk_length * sampling_rate
194
+ num_audios = mm_counts.get("audio", 0)
195
194
- audio_count = mm_counts.get("audio", 0)
195
- audio = np.zeros(audio_len)
196
- data = {"audio": [audio] * audio_count}
196
+ mm_data = {
197
+ "audio":
198
+ self._get_dummy_audios(length=audio_len, num_audios=num_audios)
199
+ }
200
201
return ProcessorInputs(
199
- prompt_text="<|audio|>" * audio_count,
200
- mm_data=data,
202
+ prompt_text="<|audio|>" * num_audios,
203
+ mm_data=mm_data,
204
)
205
206
vllm/multimodal/processing.py
+55
-13
@@ -1,15 +1,17 @@
1
import pickle
2
import re
3
from abc import ABC, abstractmethod
4
+from collections import defaultdict
5
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
6
from dataclasses import dataclass, field
7
from functools import lru_cache
8
from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
9
10
import numpy as np
11
+import numpy.typing as npt
12
import torch
13
from blake3 import blake3
12
-from PIL.Image import Image
14
+from PIL import Image
15
from transformers import BatchFeature, ProcessorMixin
16
17
from vllm.inputs import DummyData, InputProcessingContext
@@ -353,13 +355,13 @@ def _replace_matches(
355
) -> list[_S]:
356
out_seqs = list[_S]()
357
prev_end_idx = 0
356
- next_idx_by_modality = {modality: 0 for modality in mm_item_counts}
358
+ next_idx_by_modality = defaultdict[str, int](lambda: 0)
359
360
for match in _resolve_matches(prompt, matches):
361
modality = match.modality
362
363
item_idx = next_idx_by_modality[modality]
362
- if item_idx >= mm_item_counts[modality]:
364
+ if item_idx >= mm_item_counts.get(modality, 0):
365
continue
366
367
start_idx = match.start_idx
@@ -513,7 +515,7 @@ class ProcessingCache:
515
return obj.encode("utf-8")
516
if isinstance(obj, bytes):
517
return obj
516
- if isinstance(obj, Image):
518
+ if isinstance(obj, Image.Image):
519
return obj.tobytes()
520
521
# Convertible to NumPy arrays
@@ -673,10 +675,14 @@ class BaseMultiModalProcessor(ABC):
675
Given the original multi-modal items for this modality
676
and HF-processed data, output the replacements to perform.
677
676
- Note:
677
- Even when the HF processor already performs replacement for us,
678
- we still use this replacement information to determine
679
- the placeholder token positions for each multi-modal item.
678
+ Notes:
679
+ - You should not assume that HF processor always performs prompt
680
+ replacement: in :meth:`_apply_hf_processor_missing`, this method
681
+ is called on text-only and multimodal-only inputs separately,
682
+ instead of passing them in the same call.
683
+ - The replacement information returned by this method is also used
684
+ to determine the placeholder token positions for each multi-modal
685
+ item.
686
"""
687
raise NotImplementedError
688
@@ -710,6 +716,10 @@ class BaseMultiModalProcessor(ABC):
716
mm_data: Mapping[str, object],
717
mm_kwargs: Mapping[str, object],
718
) -> BatchFeature:
719
+ """
720
+ Call the HF processor on the prompt text and
721
+ associated multi-modal data.
722
+ """
723
return self.ctx.call_hf_processor(
724
self._get_hf_processor(**mm_kwargs),
725
dict(text=prompt, **mm_data),
@@ -723,7 +733,8 @@ class BaseMultiModalProcessor(ABC):
733
hf_processor_mm_kwargs: Mapping[str, object],
734
) -> tuple[list[int], MultiModalKwargs]:
735
"""
726
- Apply the HF processor on the full prompt text and multi-modal data.
736
+ Wrapper of :meth:`_call_hf_processor` that applies
737
+ additional pre-processing and post-processing.
738
"""
739
processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
740
@@ -754,10 +765,11 @@ class BaseMultiModalProcessor(ABC):
765
Apply the HF processor on the full prompt text, but only on the
766
multi-modal data that are missing from the cache.
767
757
- Note: We pass prompt text and multi-modal data into the HF processor
758
- in separate calls to avoid HF prompt replacement being done for
759
- cached items; instead, we rely on our own prompt replacement logic
760
- for the full text.
768
+ Note:
769
+ We pass prompt text and multi-modal data into the HF processor
770
+ in separate calls to avoid HF prompt replacement being done for
771
+ cached items; instead, we rely on our own prompt replacement logic
772
+ (:meth:`_get_prompt_replacements`) for the full text.
773
"""
774
mm_missing_counts = mm_missing_data_items.get_all_counts()
775
@@ -1010,6 +1022,36 @@ class BaseMultiModalProcessor(ABC):
1022
mm_placeholders=mm_placeholders,
1023
)
1024
1025
+ def _get_dummy_audios(
1026
+ self,
1027
+ *,
1028
+ length: int,
1029
+ num_audios: int,
1030
+ ) -> list[npt.NDArray]:
1031
+ audio = np.zeros((length, ))
1032
+ return [audio] * num_audios
1033
+
1034
+ def _get_dummy_images(
1035
+ self,
1036
+ *,
1037
+ width: int,
1038
+ height: int,
1039
+ num_images: int,
1040
+ ) -> list[Image.Image]:
1041
+ image = Image.new("RGB", (width, height), color=0)
1042
+ return [image] * num_images
1043
+
1044
+ def _get_dummy_videos(
1045
+ self,
1046
+ *,
1047
+ width: int,
1048
+ height: int,
1049
+ num_frames: int,
1050
+ num_videos: int,
1051
+ ) -> list[npt.NDArray]:
1052
+ video = np.zeros((num_frames, width, height, 3))
1053
+ return [video] * num_videos
1054
+
1055
@abstractmethod
1056
def _get_dummy_mm_inputs(
1057
self,
vllm/multimodal/utils.py
+7
-3
@@ -400,15 +400,19 @@ def repeat_and_pad_placeholder_tokens(
400
placeholder_token_idx = 0
401
for i, token in enumerate(prompt_token_ids):
402
if token == placeholder_token_id:
403
+ curr_repeat_count = repeat_count[placeholder_token_idx]
404
replacement_ids = repeat_and_pad_token(
405
placeholder_token_id,
405
- repeat_count=repeat_count[placeholder_token_idx],
406
+ repeat_count=curr_repeat_count,
407
pad_token_left=pad_token_left,
408
pad_token_right=pad_token_right,
409
)
410
+ offset = len(new_token_ids)
411
+ if pad_token_left is not None:
412
+ offset += 1
413
placeholder_ranges.append({
410
- "offset": len(new_token_ids),
411
- "length": len(replacement_ids)
414
+ "offset": offset,
415
+ "length": curr_repeat_count,
416
})
417
new_token_ids.extend(replacement_ids)
418
placeholder_token_idx += 1
vllm/v1/worker/gpu_model_runner.py
+14
-1
@@ -647,10 +647,23 @@ class GPUModelRunner:
647
self.mm_registry.get_max_tokens_per_item_by_modality(
648
self.model_config).values())
649
650
- max_num_mm_items = min(
650
+ max_num_mm_items_encoder_budget = min(
651
self.max_num_encoder_input_tokens,
652
self.encoder_cache_size) // max_tokens_per_mm_item
653
654
+ max_mm_items_per_req = max(
655
+ self.mm_registry.get_mm_limits_per_prompt(
656
+ self.model_config).values())
657
+
658
+ # NOTE: We do not consider max_num_batched_tokens on purpose
659
+ # because the multimodal embeddings can be generated in advance
660
+ # and chunked prefilled.
661
+ max_num_mm_items_decoder_budget = self.max_num_reqs * \
662
+ max_mm_items_per_req
663
+
664
+ max_num_mm_items = min(max_num_mm_items_encoder_budget,
665
+ max_num_mm_items_decoder_budget)
666
+
667
# Dummy data definition in V0 may contain multiple multimodal items
668
# (e.g, multiple images) for a single request, therefore here we
669
# always replicate first item by max_num_mm_items times since in V1