@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7c7c5e822
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[V1][VLM] V1 support for selected single-image models. (#11632)

Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Isotr0py <2037008807@qq.com>

Roger Wang committed Dec 31, 2024 at 13:17 UTC e7c7c5e822a886e3dba202ca1b756c3260efffcc
19 files changed +590 -636
docs/source/models/supported_models.md
+5 -5
@@ -570,28 +570,28 @@ See [this page](#generative-models) for more information on how to use generativ
570 - `rhymes-ai/Aria`
571 -
572 - ✅︎
573 - -
573 + - ✅︎
574 * - `Blip2ForConditionalGeneration`
575 - BLIP-2
576 - T + I<sup>E</sup>
577 - `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc.
578 -
579 - ✅︎
580 - -
580 + - ✅︎
581 * - `ChameleonForConditionalGeneration`
582 - Chameleon
583 - T + I
584 - `facebook/chameleon-7b` etc.
585 -
586 - ✅︎
587 - -
587 + - ✅︎
588 * - `FuyuForCausalLM`
589 - Fuyu
590 - T + I
591 - `adept/fuyu-8b` etc.
592 -
593 - ✅︎
594 - -
594 + - ✅︎
595 * - `ChatGLMModel`
596 - GLM-4V
597 - T + I
@@ -633,7 +633,7 @@ See [this page](#generative-models) for more information on how to use generativ
633 - `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, etc.
634 -
635 - ✅︎
636 - -
636 + - ✅︎
637 * - `LlavaNextVideoForConditionalGeneration`
638 - LLaVA-NeXT-Video
639 - T + V
examples/offline_inference_vision_language.py
+8 -2
@@ -24,10 +24,13 @@ def run_aria(question: str, modality: str):
24 assert modality == "image"
25 model_name = "rhymes-ai/Aria"
26
27 + # NOTE: Need L40 (or equivalent) to avoid OOM
28 llm = LLM(model=model_name,
29 tokenizer_mode="slow",
29 - trust_remote_code=True,
30 dtype="bfloat16",
31 + max_model_len=4096,
32 + max_num_seqs=2,
33 + trust_remote_code=True,
34 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
35
36 prompt = (f"<|im_start|>user\n<fim_prefix><|img|><fim_suffix>\n{question}"
@@ -57,6 +60,7 @@ def run_chameleon(question: str, modality: str):
60 prompt = f"{question}<image>"
61 llm = LLM(model="facebook/chameleon-7b",
62 max_model_len=4096,
63 + max_num_seqs=2,
64 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
65 stop_token_ids = None
66 return llm, prompt, stop_token_ids
@@ -257,7 +261,7 @@ def run_minicpmv(question: str, modality: str):
261 # 2.5
262 # model_name = "openbmb/MiniCPM-Llama3-V-2_5"
263
260 - #2.6
264 + # 2.6
265 model_name = "openbmb/MiniCPM-V-2_6"
266 tokenizer = AutoTokenizer.from_pretrained(model_name,
267 trust_remote_code=True)
@@ -430,9 +434,11 @@ def run_pixtral_hf(question: str, modality: str):
434
435 model_name = "mistral-community/pixtral-12b"
436
437 + # NOTE: Need L40 (or equivalent) to avoid OOM
438 llm = LLM(
439 model=model_name,
440 max_model_len=8192,
441 + max_num_seqs=2,
442 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
443 )
444
tests/models/decoder_only/vision_language/test_models.py
+2 -5
@@ -140,10 +140,7 @@ VLM_TEST_SETTINGS = {
140 "aria": VLMTestInfo(
141 models=["rhymes-ai/Aria"],
142 tokenizer_mode="slow",
143 - test_type=(
144 - VLMTestType.IMAGE,
145 - VLMTestType.MULTI_IMAGE,
146 - ),
143 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
144 dtype="bfloat16",
145 prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
146 img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
@@ -179,6 +176,7 @@ VLM_TEST_SETTINGS = {
176 test_type=VLMTestType.IMAGE,
177 prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
178 max_model_len=4096,
179 + max_num_seqs=2,
180 auto_cls=AutoModelForVision2Seq,
181 postprocess_inputs=model_utils.cast_dtype_post_processor(
182 "pixel_values"
@@ -201,7 +199,6 @@ VLM_TEST_SETTINGS = {
199 vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
200 num_logprobs=10,
201 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
204 - marks=[large_gpu_mark(min_gb=48)],
202 ),
203 "glm4": VLMTestInfo(
204 models=["THUDM/glm-4v-9b"],
tests/multimodal/test_processing.py
+16 -13
@@ -528,7 +528,7 @@ def _rand_audio(
528
529 def _test_processing_cache_correctness(
530 model_id: str,
531 - modalities: set[str],
531 + modalities: dict[str, bool],
532 hit_rate: float,
533 num_batches: int,
534 simplify_rate: float,
@@ -583,9 +583,8 @@ def _test_processing_cache_correctness(
583 partial(_rand_audio, rng, min_len=256, max_len=512, sr=16000),
584 }
585 input_max_count = {
586 - "image": 3,
587 - "video": 3,
588 - "audio": 3,
586 + modality: 3 if supports_multi else 1
587 + for modality, supports_multi in modalities.items()
588 }
589
590 for batch_idx in range(num_batches):
@@ -624,12 +623,16 @@ def _test_processing_cache_correctness(
623
624 # yapf: disable
625 @pytest.mark.parametrize(("model_id", "modalities"), [
627 - ("llava-hf/llava-1.5-7b-hf", {"image"}),
628 - ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image"}),
629 - ("mistral-community/pixtral-12b", {"image"}),
630 - ("Qwen/Qwen2-VL-2B-Instruct", {"image", "video"}),
631 - ("Qwen/Qwen2-Audio-7B-Instruct", {"audio"}),
632 - ("fixie-ai/ultravox-v0_3", {"audio"}),
626 + ("rhymes-ai/Aria", {"image": True}),
627 + ("Salesforce/blip2-opt-2.7b", {"image": False}),
628 + ("facebook/chameleon-7b", {"image": True}),
629 + ("adept/fuyu-8b", {"image": False}),
630 + ("llava-hf/llava-1.5-7b-hf", {"image": True}),
631 + ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
632 + ("mistral-community/pixtral-12b", {"image": True}),
633 + ("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
634 + ("Qwen/Qwen2-Audio-7B-Instruct", {"audio": True}),
635 + ("fixie-ai/ultravox-v0_3", {"audio": True}),
636 ])
637 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
638 @pytest.mark.parametrize("num_batches", [32])
@@ -637,7 +640,7 @@ def _test_processing_cache_correctness(
640 # yapf: enable
641 def test_processing_cache_correctness(
642 model_id: str,
640 - modalities: set[str],
643 + modalities: dict[str, bool],
644 hit_rate: float,
645 num_batches: int,
646 simplify_rate: float,
@@ -653,7 +656,7 @@ def test_processing_cache_correctness(
656
657 # yapf: disable
658 @pytest.mark.parametrize(("model_id", "modalities"), [
656 - ("microsoft/Phi-3-vision-128k-instruct", {"image"}),
659 + ("microsoft/Phi-3-vision-128k-instruct", {"image": True}),
660 ])
661 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
662 @pytest.mark.parametrize("num_batches", [32])
@@ -661,7 +664,7 @@ def test_processing_cache_correctness(
664 # yapf: enable
665 def test_processing_cache_correctness_phi3v(
666 model_id: str,
664 - modalities: set[str],
667 + modalities: dict[str, bool],
668 hit_rate: float,
669 num_batches: int,
670 simplify_rate: float,
vllm/model_executor/models/aria.py
+80 -89
@@ -1,15 +1,15 @@
1 -import math
2 -from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
1 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
2 + Union)
3
4 import torch
5 import torch.nn as nn
6 from torch.nn.init import trunc_normal_
7 -from transformers import LlamaConfig
7 +from transformers import BatchFeature, PretrainedConfig
8
9 from vllm.attention import AttentionMetadata
10 from vllm.config import CacheConfig, QuantizationConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_rank
12 -from vllm.inputs import INPUT_REGISTRY, token_inputs
12 +from vllm.inputs import InputContext
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.fused_moe import FusedMoE
15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -17,30 +17,27 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 from vllm.model_executor.layers.logits_processor import LogitsProcessor
18 from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
19 get_compressed_tensors_cache_scale)
20 -from vllm.model_executor.layers.sampler import (Sampler, SamplerOutput,
21 - SamplingMetadata)
20 +from vllm.model_executor.layers.sampler import (SamplerOutput,
21 + SamplingMetadata, get_sampler)
22 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
23 from vllm.model_executor.model_loader.weight_utils import (
24 default_weight_loader, maybe_remap_kv_scale_name)
25 -from vllm.model_executor.models.idefics2_vision_model import (
26 - Idefics2VisionTransformer)
27 -from vllm.model_executor.models.interfaces import SupportsMultiModal
28 -from vllm.model_executor.models.llama import (LlamaDecoderLayer, LlamaMLP,
29 - LlamaModel)
30 -from vllm.model_executor.models.utils import (AutoWeightsLoader, WeightsMapper,
31 - is_pp_missing_parameter,
32 - maybe_prefix,
33 - merge_multimodal_embeddings)
25 from vllm.multimodal import MULTIMODAL_REGISTRY
35 -from vllm.multimodal.image import cached_get_image_processor
36 -from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
37 -from vllm.multimodal.utils import (cached_get_tokenizer,
38 - repeat_and_pad_placeholder_tokens)
26 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
27 + NestedTensors)
28 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 + MultiModalDataItems, ProcessorInputs,
30 + PromptReplacement)
31 from vllm.sequence import IntermediateTensors
32 from vllm.transformers_utils.configs.aria import (AriaMoELMConfig,
33 AriaVisionConfig)
34
43 -from .utils import flatten_bn
35 +from .idefics2_vision_model import Idefics2VisionTransformer
36 +from .interfaces import SupportsMultiModal
37 +from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
38 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
39 + is_pp_missing_parameter, maybe_prefix,
40 + merge_multimodal_embeddings)
41
42
43 class AriaImagePixelInputs(TypedDict):
@@ -251,7 +248,7 @@ class AriaProjector(nn.Module):
248 class AriaFusedMoE(FusedMoE):
249
250 def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor,
254 - shard_id: str) -> Set[str]:
251 + shard_id: str) -> None:
252 # Override the weight_loader to handle the expert weights in the Aria
253 # model, which are already packed with experts, and merge the gate and
254 # up weights for each expert.
@@ -346,7 +343,7 @@ class MoEDecoderLayer(LlamaDecoderLayer):
343
344 def __init__(
345 self,
349 - config: LlamaConfig,
346 + config: AriaMoELMConfig,
347 cache_config: Optional[CacheConfig] = None,
348 quant_config: Optional[QuantizationConfig] = None,
349 prefix: str = "",
@@ -434,7 +431,7 @@ class AriaMoELMModel(LlamaModel):
431 return loaded_params
432
433
437 -def build_mm_projector(config):
434 +def build_mm_projector(config: PretrainedConfig):
435 return AriaProjector(
436 patch_to_query_dict=config.projector_patch_to_query_dict,
437 embed_dim=config.vision_config.hidden_size,
@@ -445,75 +442,70 @@ def build_mm_projector(config):
442 )
443
444
448 -def get_max_multimodal_tokens(ctx):
449 - return max(ctx.model_config.hf_config.image_size2tokens.values())
450 -
451 -
452 -def input_mapper_for_aria(ctx, data):
453 - return MultiModalKwargs(data)
445 +def get_max_aria_image_tokens(ctx: InputContext):
446 + hf_config = ctx.get_hf_config()
447 + return max(hf_config.projector_patch_to_query_dict.values())
448
449
456 -def input_processor(ctx, llm_inputs):
457 - multi_modal_data = llm_inputs.get("multi_modal_data")
458 - # if it is pure text input, use it as is
459 - if multi_modal_data is None or "image" not in multi_modal_data:
460 - return llm_inputs
450 +class AriaMultiModalProcessor(BaseMultiModalProcessor):
451
462 - model_config = ctx.model_config
463 -
464 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
465 - image_processor = cached_get_image_processor(
466 - model_config.model, trust_remote_code=model_config.trust_remote_code)
467 - hf_config = model_config.hf_config
452 + def _get_mm_fields_config(
453 + self,
454 + hf_inputs: BatchFeature,
455 + hf_processor_mm_kwargs: Mapping[str, object],
456 + ) -> Mapping[str, MultiModalFieldConfig]:
457 + return dict(
458 + pixel_values=MultiModalFieldConfig.batched("image"),
459 + pixel_mask=MultiModalFieldConfig.batched("image"),
460 + )
461
469 - # prepare image tokens, the max_image_size is used to determine the number
470 - # of patch_size for every image
471 - max_image_size = multi_modal_data.pop("max_image_size", 980)
472 - _split_image = multi_modal_data.pop("split_image", False)
462 + def _get_prompt_replacements(
463 + self,
464 + mm_items: MultiModalDataItems,
465 + hf_processor_mm_kwargs: Mapping[str, object],
466 + out_mm_kwargs: MultiModalKwargs,
467 + ) -> list[PromptReplacement]:
468 + hf_config = self.ctx.get_hf_config()
469 + image_token_id = hf_config.image_token_index
470 +
471 + max_image_tokens = get_max_aria_image_tokens(self.ctx)
472 +
473 + return [
474 + PromptReplacement(
475 + modality="image",
476 + target=[image_token_id],
477 + replacement=[image_token_id] * max_image_tokens,
478 + )
479 + ]
480
474 - assert isinstance(max_image_size,
475 - (int, float)), "max_image_size should be float or int"
476 - images = (multi_modal_data["image"] if isinstance(
477 - multi_modal_data["image"], list) else [multi_modal_data["image"]])
481 + def _get_dummy_mm_inputs(
482 + self,
483 + mm_counts: Mapping[str, int],
484 + ) -> ProcessorInputs:
485 + hf_config = self.ctx.get_hf_config()
486 + vision_config: AriaVisionConfig = hf_config.vision_config
487 +
488 + max_image_size = vision_config.image_size
489 + num_images = mm_counts.get("image", 0)
490 +
491 + mm_data = {
492 + "image":
493 + self._get_dummy_images(width=max_image_size,
494 + height=max_image_size,
495 + num_images=num_images)
496 + }
497
479 - image_inputs = image_processor.preprocess(images,
480 - max_image_size=max_image_size,
481 - split_image=_split_image,
482 - return_tensors="pt").data
483 - image_inputs['pixel_values'] = image_inputs['pixel_values'].to(
484 - ctx.model_config.dtype)
485 - num_crops = image_inputs.pop("num_crops")
498 + hf_processor = self._get_hf_processor()
499 + image_token: str = hf_processor.image_token # type: ignore
500
487 - prompt_token_ids = llm_inputs["prompt_token_ids"]
488 - if num_crops.sum().item() > 0:
489 - _, prompt_token_ids, _ = repeat_and_pad_placeholder_tokens(
490 - tokenizer,
491 - None,
492 - prompt_token_ids,
493 - placeholder_token_id=hf_config.image_token_index,
494 - repeat_count=num_crops,
501 + return ProcessorInputs(
502 + prompt_text=image_token * num_images,
503 + mm_data=mm_data,
504 )
505
497 - repeat_count = [hf_config.image_size2tokens[max_image_size]
498 - ] * sum(num_crops).item()
499 - new_prompt, new_token_ids, _ = repeat_and_pad_placeholder_tokens(
500 - tokenizer,
501 - None,
502 - prompt_token_ids,
503 - placeholder_token_id=hf_config.image_token_index,
504 - repeat_count=repeat_count,
505 - )
506 -
507 - return token_inputs(
508 - prompt_token_ids=new_token_ids,
509 - prompt=new_prompt,
510 - multi_modal_data={"image": image_inputs},
511 - )
512 -
506
514 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_multimodal_tokens)
515 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_aria)
516 -@INPUT_REGISTRY.register_input_processor(input_processor)
507 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_aria_image_tokens)
508 +@MULTIMODAL_REGISTRY.register_processor(AriaMultiModalProcessor)
509 class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
510 """
511 Aria model for conditional generation tasks.
@@ -540,12 +532,6 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
532 config = vllm_config.model_config.hf_config
533 quant_config = vllm_config.quant_config
534
543 - # prepare the image_size to tokens mapping for the image preprocess, see
544 - # input_processor
545 - config.image_size2tokens = {
546 - int(math.sqrt(k) * config.vision_config.patch_size): v
547 - for k, v in config.projector_patch_to_query_dict.items()
548 - }
535 self.config = config
536 self.vision_tower = AriaVisionModel(config.vision_config)
537 self.multi_modal_projector = build_mm_projector(config)
@@ -566,7 +552,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
552 logit_scale = getattr(config, "logit_scale", 1.0)
553 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
554 self.vocab_size, logit_scale)
569 - self.sampler = Sampler()
555 + self.sampler = get_sampler()
556
557 def _validate_image_sizes(
558 self, images: List[torch.Tensor]) -> List[torch.Tensor]:
@@ -588,7 +574,12 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
574
575 pixel_values = self._validate_image_sizes(pixel_values)
576 pixel_values = flatten_bn(pixel_values, concat=True)
577 +
578 if pixel_mask is not None:
579 + if not isinstance(pixel_mask, (torch.Tensor, list)):
580 + raise ValueError("Incorrect type of pixel mask. "
581 + f"Got type: {type(pixel_mask)}")
582 +
583 pixel_mask = flatten_bn(pixel_mask, concat=True)
584
585 return AriaImagePixelInputs(
vllm/model_executor/models/blip.py
-92
@@ -4,22 +4,16 @@ from typing import Iterable, Optional, Set, Tuple, Union
4
5 import torch
6 import torch.nn as nn
7 -from PIL import Image
7 from transformers import Blip2VisionConfig, BlipVisionConfig
8
9 from vllm.attention.layer import MultiHeadAttention
11 -from vllm.config import ModelConfig
10 from vllm.distributed import divide, get_tensor_model_parallel_world_size
13 -from vllm.inputs import DecoderOnlyInputs, token_inputs
11 from vllm.model_executor.layers.activation import get_act_fn
12 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13 QKVParallelLinear,
14 RowParallelLinear)
15 from vllm.model_executor.layers.quantization import QuantizationConfig
16 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 -from vllm.multimodal.utils import (cached_get_tokenizer,
21 - repeat_and_pad_placeholder_tokens)
22 -from vllm.sequence import SequenceData
17
18
19 def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -33,92 +27,6 @@ def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
27 return grid_length * grid_length
28
29
36 -def get_blip_image_feature_size(
37 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
38 - return get_blip_num_patches(image_size=hf_config.image_size,
39 - patch_size=hf_config.patch_size)
40 -
41 -
42 -def get_max_blip_image_tokens(
43 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig]) -> int:
44 - return get_blip_image_feature_size(hf_config)
45 -
46 -
47 -def dummy_seq_data_for_blip(
48 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
49 - seq_len: int,
50 - num_images: int,
51 - *,
52 - image_token_id: int,
53 - image_feature_size_override: Optional[int] = None,
54 -):
55 - if image_feature_size_override is None:
56 - image_feature_size = get_blip_image_feature_size(hf_config)
57 - else:
58 - image_feature_size = image_feature_size_override
59 -
60 - return SequenceData.from_prompt_token_counts(
61 - (image_token_id, image_feature_size * num_images),
62 - (0, seq_len - image_feature_size * num_images),
63 - )
64 -
65 -
66 -def dummy_image_for_blip(
67 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
68 - num_images: int,
69 - *,
70 - image_width_override: Optional[int] = None,
71 - image_height_override: Optional[int] = None,
72 -):
73 - width = height = hf_config.image_size
74 - if image_width_override is not None:
75 - width = image_width_override
76 - if image_height_override is not None:
77 - height = image_height_override
78 -
79 - image = Image.new("RGB", (width, height), color=0)
80 - return {"image": image if num_images == 1 else [image] * num_images}
81 -
82 -
83 -def input_processor_for_blip(
84 - model_config: ModelConfig,
85 - hf_config: Union[BlipVisionConfig, Blip2VisionConfig],
86 - inputs: DecoderOnlyInputs,
87 - *,
88 - image_token_id: int,
89 - image_feature_size_override: Optional[int] = None,
90 -):
91 - multi_modal_data = inputs.get("multi_modal_data")
92 - if multi_modal_data is None or "image" not in multi_modal_data:
93 - return inputs
94 -
95 - if "multi_modal_placeholders" in inputs and "image" in inputs[
96 - "multi_modal_placeholders"]:
97 - # The inputs already have placeholders.
98 - return inputs
99 -
100 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
101 -
102 - if image_feature_size_override is None:
103 - image_feature_size = get_blip_image_feature_size(hf_config)
104 - else:
105 - image_feature_size = image_feature_size_override
106 -
107 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
108 - tokenizer,
109 - inputs.get("prompt"),
110 - inputs["prompt_token_ids"],
111 - placeholder_token_id=image_token_id,
112 - repeat_count=image_feature_size,
113 - )
114 -
115 - # NOTE: Create a defensive copy of the original inputs
116 - return token_inputs(prompt_token_ids=new_token_ids,
117 - prompt=new_prompt,
118 - multi_modal_data=multi_modal_data,
119 - multi_modal_placeholders={"image": ranges})
120 -
121 -
30 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
31 class BlipVisionEmbeddings(nn.Module):
32
vllm/model_executor/models/blip2.py
+84 -88
@@ -4,32 +4,33 @@ from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
5 import torch
6 import torch.nn as nn
7 -from transformers import (Blip2Config, Blip2QFormerConfig, Blip2VisionConfig,
8 - apply_chunking_to_forward)
7 +from transformers import (BatchFeature, Blip2Config, Blip2Processor,
8 + Blip2QFormerConfig, apply_chunking_to_forward)
9
10 from vllm.attention import AttentionMetadata
11 from vllm.config import CacheConfig, VllmConfig
12 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
13 - InputContext, token_inputs)
12 +from vllm.inputs import InputContext
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.quantization import QuantizationConfig
15 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
19 -from vllm.multimodal.inputs import NestedTensors
20 -from vllm.multimodal.utils import consecutive_placeholder_ranges
21 -from vllm.sequence import IntermediateTensors, SequenceData
22 -
23 -from .blip import (BlipVisionModel, dummy_image_for_blip,
24 - get_max_blip_image_tokens)
18 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
19 + MultiModalInputsV2, MultiModalKwargs,
20 + NestedTensors, PlaceholderRange)
21 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
22 + MultiModalDataItems, ProcessorInputs,
23 + PromptReplacement)
24 +from vllm.sequence import IntermediateTensors
25 +
26 +from .blip import BlipVisionModel
27 from .interfaces import SupportsMultiModal, SupportsPP
28 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
29 maybe_prefix, merge_multimodal_embeddings)
30
31 # We use this internally as placeholders since there is no image token
32 # defined on the HuggingFace repo
31 -BLIP2_IMAGE_TOKEN = "<image>"
32 -BLIP2_IMAGE_TOKEN_ID = 50265
33 +_IMAGE_TOKEN_ID = 50265
34
35
36 class Blip2ImagePixelInputs(TypedDict):
@@ -396,92 +397,87 @@ class Blip2QFormerModel(nn.Module):
397 return sequence_output
398
399
399 -def get_blip2_image_feature_size(hf_config: Blip2Config) -> int:
400 - return hf_config.num_query_tokens
401 -
402 -
400 def get_max_blip2_image_tokens(ctx: InputContext):
401 hf_config = ctx.get_hf_config(Blip2Config)
405 - vision_config = hf_config.vision_config
406 -
407 - if isinstance(vision_config, Blip2VisionConfig):
408 - return get_max_blip_image_tokens(vision_config)
409 -
410 - msg = f"Unsupported vision config: {type(vision_config)}"
411 - raise NotImplementedError(msg)
412 -
413 -
414 -def dummy_seq_data_for_blip2(
415 - hf_config: Blip2Config,
416 - seq_len: int,
417 - num_images: int,
418 - *,
419 - image_token_id: int,
420 - image_feature_size_override: Optional[int] = None,
421 -):
422 - if image_feature_size_override is None:
423 - image_feature_size = get_blip2_image_feature_size(hf_config)
424 - else:
425 - image_feature_size = image_feature_size_override
426 -
427 - return SequenceData.from_prompt_token_counts(
428 - (image_token_id, image_feature_size * num_images),
429 - (0, seq_len - image_feature_size * num_images),
430 - ), {
431 - "image":
432 - consecutive_placeholder_ranges(num_items=num_images,
433 - item_size=image_feature_size)
434 - }
435 -
436 -
437 -def dummy_data_for_blip2(ctx: InputContext, seq_len: int,
438 - mm_counts: Mapping[str, int]):
439 - hf_config = ctx.get_hf_config(Blip2Config)
440 - vision_config = hf_config.vision_config
441 - num_images = mm_counts["image"]
442 -
443 - seq_data, ranges = dummy_seq_data_for_blip2(
444 - hf_config,
445 - seq_len,
446 - num_images,
447 - image_token_id=BLIP2_IMAGE_TOKEN_ID,
448 - )
449 -
450 - if isinstance(vision_config, Blip2VisionConfig):
451 - mm_data = dummy_image_for_blip(vision_config, num_images)
452 -
453 - return DummyData(seq_data, mm_data, ranges)
454 -
455 - msg = f"Unsupported vision config: {type(vision_config)}"
456 - raise NotImplementedError(msg)
402 + return hf_config.num_query_tokens
403
404
459 -def input_processor_for_blip2(ctx: InputContext, inputs: DecoderOnlyInputs):
460 - multi_modal_data = inputs.get("multi_modal_data")
461 - if multi_modal_data is None or "image" not in multi_modal_data:
462 - return inputs
405 +class Blip2MultiModalProcessor(BaseMultiModalProcessor):
406
464 - hf_config = ctx.get_hf_config(Blip2Config)
465 - image_feature_size = get_blip2_image_feature_size(hf_config)
407 + def _get_hf_processor(self) -> Blip2Processor:
408 + return self.ctx.get_hf_processor(Blip2Processor)
409
467 - # The original model places image tokens at the front
468 - # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1514
469 - new_token_ids = [BLIP2_IMAGE_TOKEN_ID] * image_feature_size
470 - new_token_ids += inputs["prompt_token_ids"]
410 + def _get_mm_fields_config(
411 + self,
412 + hf_inputs: BatchFeature,
413 + hf_processor_mm_kwargs: Mapping[str, object],
414 + ) -> Mapping[str, MultiModalFieldConfig]:
415 + return dict(
416 + pixel_values=MultiModalFieldConfig.batched("image"),
417 + image_embeds=MultiModalFieldConfig.batched("image"),
418 + )
419
472 - new_prompt = inputs.get("prompt")
473 - if new_prompt is not None:
474 - new_prompt = BLIP2_IMAGE_TOKEN * image_feature_size + new_prompt
420 + def _get_prompt_replacements(
421 + self,
422 + mm_items: MultiModalDataItems,
423 + hf_processor_mm_kwargs: Mapping[str, object],
424 + out_mm_kwargs: MultiModalKwargs,
425 + ) -> list[PromptReplacement]:
426 + max_image_tokens = get_max_blip2_image_tokens(self.ctx)
427 +
428 + return [
429 + PromptReplacement(
430 + modality="image",
431 + target="</s>",
432 + replacement="<image>" * max_image_tokens + "</s>",
433 + )
434 + ]
435
476 - return token_inputs(prompt_token_ids=new_token_ids,
477 - prompt=new_prompt,
478 - multi_modal_data=multi_modal_data)
436 + def apply(
437 + self,
438 + prompt_text: str,
439 + mm_data: MultiModalDataDict,
440 + hf_processor_mm_kwargs: Mapping[str, object],
441 + ) -> MultiModalInputsV2:
442 + result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
443 +
444 + # Only <image> tokens should be considered as placeholders,
445 + # so we ignore the trailing bos_token
446 + result["mm_placeholders"] = {
447 + modality: [
448 + PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
449 + for p in ps
450 + ]
451 + for modality, ps in result["mm_placeholders"].items()
452 + }
453 +
454 + return result
455 +
456 + def _get_dummy_mm_inputs(
457 + self,
458 + mm_counts: Mapping[str, int],
459 + ) -> ProcessorInputs:
460 + hf_config = self.ctx.get_hf_config(Blip2Config)
461 + vision_config = hf_config.vision_config
462 +
463 + max_image_size = vision_config.image_size
464 + num_images = mm_counts.get("image", 0)
465 +
466 + mm_data = {
467 + "image":
468 + self._get_dummy_images(width=max_image_size,
469 + height=max_image_size,
470 + num_images=num_images)
471 + }
472 +
473 + return ProcessorInputs(
474 + prompt_text="",
475 + mm_data=mm_data,
476 + )
477
478
481 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
479 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_blip2_image_tokens)
483 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_blip2)
484 -@INPUT_REGISTRY.register_input_processor(input_processor_for_blip2)
480 +@MULTIMODAL_REGISTRY.register_processor(Blip2MultiModalProcessor)
481 class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
482
483 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -627,7 +623,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
623 if multimodal_embeddings is not None:
624 inputs_embeds = merge_multimodal_embeddings(
625 input_ids, inputs_embeds, multimodal_embeddings,
630 - BLIP2_IMAGE_TOKEN_ID)
626 + _IMAGE_TOKEN_ID)
627 return inputs_embeds
628
629 def forward(
vllm/model_executor/models/chameleon.py
+81 -110
@@ -3,16 +3,15 @@ from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
3 Tuple, TypedDict, Union)
4
5 import torch
6 +import torch.nn as nn
7 import torch.nn.functional as F
7 -from PIL import Image
8 -from torch import nn
9 -from transformers import ChameleonConfig, ChameleonVQVAEConfig
8 +from transformers import (BatchFeature, ChameleonConfig, ChameleonProcessor,
9 + ChameleonVQVAEConfig)
10
11 from vllm.attention import Attention, AttentionMetadata
12 from vllm.config import CacheConfig, VllmConfig
13 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
14 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 - InputContext, token_inputs)
14 +from vllm.inputs import InputContext
15 from vllm.model_executor.layers.activation import SiluAndMul
16 from vllm.model_executor.layers.layernorm import RMSNorm
17 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -29,11 +28,13 @@ from vllm.model_executor.model_loader.weight_utils import (
28 from vllm.model_executor.sampling_metadata import SamplingMetadata
29 from vllm.model_executor.utils import set_weight_attrs
30 from vllm.multimodal import MULTIMODAL_REGISTRY
32 -from vllm.multimodal.inputs import NestedTensors
33 -from vllm.multimodal.utils import (cached_get_tokenizer,
34 - consecutive_placeholder_ranges,
35 - repeat_and_pad_placeholder_tokens)
36 -from vllm.sequence import IntermediateTensors, SequenceData
31 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32 + MultiModalInputsV2, MultiModalKwargs,
33 + NestedTensors, PlaceholderRange)
34 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
35 + MultiModalDataItems, ProcessorInputs,
36 + PromptReplacement)
37 +from vllm.sequence import IntermediateTensors
38 from vllm.utils import print_warning_once
39
40 from .interfaces import SupportsMultiModal, SupportsPP
@@ -45,10 +46,6 @@ from .utils import (is_pp_missing_parameter,
46 # and processor files, so we hardcode them in the model file for now.
47 CHAMELEON_CROP_SIZE_HEIGHT = CHAMELEON_CROP_SIZE_WIDTH = 512
48 CHAMELEON_IMAGE_SEQ_LENGTH = 1024
48 -CHAMELEON_IMAGE_TOKEN_ID = 8711
49 -CHAMELEON_IMAGE_START_TOKEN_ID = 8197
50 -CHAMELEON_IMAGE_END_TOKEN_ID = 8196
51 -CHAMELEON_SEP_TOKEN_ID = 8710
49
50
51 class ChameleonImagePixelInputs(TypedDict):
@@ -61,99 +58,75 @@ def get_max_chameleon_image_tokens(ctx: InputContext):
58 return CHAMELEON_IMAGE_SEQ_LENGTH
59
60
64 -def dummy_seq_data_for_chameleon(
65 - seq_len: int,
66 - num_images: int,
67 - *,
68 - image_token_id: int,
69 - image_feature_size_override: Optional[int] = None,
70 -):
71 - if image_feature_size_override is None:
72 - image_feature_size = CHAMELEON_IMAGE_SEQ_LENGTH
73 - else:
74 - image_feature_size = image_feature_size_override
75 -
76 - return SequenceData.from_prompt_token_counts(
77 - (image_token_id, image_feature_size * num_images),
78 - (0, seq_len - image_feature_size * num_images),
79 - ), {
80 - "image":
81 - consecutive_placeholder_ranges(num_items=num_images,
82 - item_size=image_feature_size)
83 - }
84 -
85 -
86 -def dummy_image_for_chameleon(
87 - num_images: int,
88 - *,
89 - image_width_override: Optional[int] = None,
90 - image_height_override: Optional[int] = None,
91 -):
92 - width = CHAMELEON_CROP_SIZE_WIDTH
93 - height = CHAMELEON_CROP_SIZE_HEIGHT
94 - if image_width_override is not None:
95 - width = image_width_override
96 - if image_height_override is not None:
97 - height = image_height_override
98 -
99 - image = Image.new("RGB", (width, height), color=0)
100 - return {"image": image if num_images == 1 else [image] * num_images}
101 -
102 -
103 -def dummy_data_for_chameleon(ctx: InputContext, seq_len: int,
104 - mm_counts: Mapping[str, int]):
105 - num_images = mm_counts["image"]
106 -
107 - seq_data, ranges = dummy_seq_data_for_chameleon(
108 - seq_len,
109 - num_images,
110 - image_token_id=CHAMELEON_IMAGE_TOKEN_ID,
111 - )
112 -
113 - mm_data = dummy_image_for_chameleon(num_images)
114 - return DummyData(seq_data, mm_data, ranges)
115 -
116 -
117 -def input_processor_for_chameleon(ctx: InputContext,
118 - inputs: DecoderOnlyInputs):
61 +class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
62
120 - """
121 - Processing input prompt to insert required tokens for image placeholder.
122 -
123 - See https://github.com/huggingface/transformers/blob/0fdea8607d7e01eb0e38a1ebeb7feee30a22f0cf/src/transformers/models/chameleon/processing_chameleon.py#L58
124 - """ # noqa
125 -
126 - multi_modal_data = inputs.get("multi_modal_data")
127 - if multi_modal_data is None or "image" not in multi_modal_data:
128 - return inputs
129 -
130 - if "multi_modal_placeholders" in inputs and "image" in inputs[
131 - "multi_modal_placeholders"]:
132 - # The inputs already have placeholders.
133 - return inputs
134 -
135 - model_config = ctx.model_config
136 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
137 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
138 - tokenizer,
139 - inputs.get("prompt"),
140 - inputs["prompt_token_ids"],
141 - placeholder_token_id=CHAMELEON_IMAGE_TOKEN_ID,
142 - repeat_count=CHAMELEON_IMAGE_SEQ_LENGTH,
143 - pad_token_left=CHAMELEON_IMAGE_START_TOKEN_ID,
144 - pad_token_right=CHAMELEON_IMAGE_END_TOKEN_ID,
145 - )
146 -
147 - # Appending sep token for chat mode to follow default processor
148 - # behavior
149 - if new_prompt is not None:
150 - new_prompt += tokenizer.sep_token
151 - new_token_ids += [CHAMELEON_SEP_TOKEN_ID]
152 -
153 - # NOTE: Create a defensive copy of the original inputs
154 - return token_inputs(prompt_token_ids=new_token_ids,
155 - prompt=new_prompt,
156 - multi_modal_data=multi_modal_data)
63 + def _get_hf_processor(self) -> ChameleonProcessor:
64 + return self.ctx.get_hf_processor(ChameleonProcessor)
65 +
66 + def _get_mm_fields_config(
67 + self,
68 + hf_inputs: BatchFeature,
69 + hf_processor_mm_kwargs: Mapping[str, object],
70 + ) -> Mapping[str, MultiModalFieldConfig]:
71 + return dict(pixel_values=MultiModalFieldConfig.batched("image"))
72 +
73 + def _get_prompt_replacements(
74 + self,
75 + mm_items: MultiModalDataItems,
76 + hf_processor_mm_kwargs: Mapping[str, object],
77 + out_mm_kwargs: MultiModalKwargs,
78 + ) -> list[PromptReplacement]:
79 + processor = self._get_hf_processor()
80 +
81 + return [
82 + PromptReplacement(
83 + modality="image",
84 + target="<image>",
85 + replacement="".join([
86 + processor.image_start_token,
87 + processor.image_token * CHAMELEON_IMAGE_SEQ_LENGTH,
88 + processor.image_end_token,
89 + ]),
90 + )
91 + ]
92 +
93 + def _get_dummy_mm_inputs(
94 + self,
95 + mm_counts: Mapping[str, int],
96 + ) -> ProcessorInputs:
97 + num_images = mm_counts.get("image", 0)
98 +
99 + mm_data = {
100 + "image":
101 + self._get_dummy_images(width=CHAMELEON_CROP_SIZE_WIDTH,
102 + height=CHAMELEON_CROP_SIZE_HEIGHT,
103 + num_images=num_images)
104 + }
105 +
106 + return ProcessorInputs(
107 + prompt_text="<image>" * num_images,
108 + mm_data=mm_data,
109 + )
110 +
111 + def apply(
112 + self,
113 + prompt_text: str,
114 + mm_data: MultiModalDataDict,
115 + hf_processor_mm_kwargs: Mapping[str, object],
116 + ) -> MultiModalInputsV2:
117 + result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
118 +
119 + # Only <image> tokens should be considered as placeholders,
120 + # so we ignore the image_start_token and image_end_token
121 + result["mm_placeholders"] = {
122 + modality: [
123 + PlaceholderRange(offset=p["offset"] + 1,
124 + length=p["length"] - 2) for p in ps
125 + ]
126 + for modality, ps in result["mm_placeholders"].items()
127 + }
128 +
129 + return result
130
131
132 class ChameleonLayerNorm(nn.LayerNorm):
@@ -736,7 +709,7 @@ class ChameleonVQVAEEncoder(nn.Module):
709 for i_level in range(self.num_resolutions):
710 for i_block in range(self.num_res_blocks):
711 hidden_state = self.down[i_level].block[i_block](
739 - hidden_states[-1], )
712 + hidden_states[-1])
713 if len(self.down[i_level].attn) > 0:
714 hidden_state = self.down[i_level].attn[i_block](
715 hidden_state)
@@ -925,10 +898,8 @@ class ChameleonModel(nn.Module):
898 return hidden_states
899
900
928 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
901 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_chameleon_image_tokens)
930 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_chameleon)
931 -@INPUT_REGISTRY.register_input_processor(input_processor_for_chameleon)
902 +@MULTIMODAL_REGISTRY.register_processor(ChameleonMultiModalProcessor)
903 class ChameleonForConditionalGeneration(nn.Module, SupportsMultiModal,
904 SupportsPP):
905
vllm/model_executor/models/fuyu.py
+199 -182
@@ -15,32 +15,30 @@
15 # limitations under the License.
16 """ PyTorch Fuyu model."""
17 import math
18 -from array import array
18 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
19 TypedDict)
20
21 import torch
22 import torch.nn as nn
24 -import torch.utils.checkpoint
25 -from PIL import Image
26 -from transformers import FuyuImageProcessor
23 +from transformers import (BatchFeature, FuyuConfig, FuyuImageProcessor,
24 + FuyuProcessor)
25
26 from vllm.attention import AttentionMetadata
27 from vllm.config import VllmConfig
30 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
31 - InputContext, token_inputs)
28 +from vllm.inputs import InputContext
29 from vllm.model_executor.layers.linear import ColumnParallelLinear
30 from vllm.model_executor.layers.sampler import SamplerOutput
31 from vllm.model_executor.models.persimmon import PersimmonForCausalLM
32 from vllm.model_executor.sampling_metadata import SamplingMetadata
36 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
37 -from vllm.multimodal.image import cached_get_image_processor
38 -from vllm.multimodal.inputs import NestedTensors
39 -from vllm.multimodal.utils import (cached_get_tokenizer,
40 - consecutive_placeholder_ranges)
41 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 - SequenceData)
43 -from vllm.utils import is_list_of
33 +from vllm.multimodal import MULTIMODAL_REGISTRY
34 +from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35 + MultiModalInputsV2, MultiModalKwargs,
36 + NestedTensors, PlaceholderRange)
37 +from vllm.multimodal.parse import ImageProcessorItems
38 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 + MultiModalDataItems, ProcessorInputs,
40 + PromptReplacement)
41 +from vllm.sequence import IntermediateTensors
42
43 from .interfaces import SupportsMultiModal, SupportsPP
44 from .utils import (AutoWeightsLoader, flatten_bn, maybe_prefix,
@@ -54,178 +52,193 @@ MAX_IMAGE_FEATURE_SIZE_HEIGHT = 1080
52 MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
53
54
57 -class FuyuImagePixelInputs(TypedDict):
58 - type: Literal["pixel_values"]
55 +class FuyuImagePatchInputs(TypedDict):
56 + type: Literal["image_patches"]
57 data: torch.Tensor
58 """
59 Shape:
62 - (batch_size, num_patches, patch_size_x * patch_size_y * num_channels)
60 + `(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`
61 + """
62 +
63 + patches_per_image: List[int]
64 + """
65 + List of number of total patches for each image in the batch.
66 + This is used to restore the first two dimensions of `data`.
67 """
68
69
66 -def _calculate_num_image_tokens(
67 - height: int,
68 - width: int,
70 +def _get_fuyu_num_image_tokens(
71 + image_height: int,
72 + image_width: int,
73 ) -> Tuple[int, int]:
74 """
71 - calculate number of image tokens needed for a given image size
72 - The expected Fuyu image prompts is in format:
73 - (image_token * ncols + newline_token) * nrows
74 - args:
75 - image_size: Tuple[int, int] - (width, height) of the image
76 - returns:
77 - ncols: int - number of image tokens in x direction
78 - nrows: int - number of image tokens in y direction
79 - """
80 - ncol = math.ceil(width / 30)
81 - nrow = math.ceil(height / 30)
82 - return ncol, nrow
75 + Calculate the number of image tokens needed for a given image size.
76
77 + The expected Fuyu image prompts can be expressed as:
78
85 -def get_max_fuyu_image_feature_size():
79 + .. code-block::
80 + (image_token * ncols + newline_token) * nrows
81
87 - return _calculate_num_image_tokens(
88 - height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
89 - width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
90 - )
82 + Args:
83 + image_size: Tuple[int, int] - `(width, height)` of the image
84 +
85 + Returns:
86 + ncols: int - number of image tokens in `x` direction
87 + nrows: int - number of image tokens in `y` direction
88 + """
89 + ncols = math.ceil(image_width / 30)
90 + nrows = math.ceil(image_height / 30)
91 + return ncols, nrows
92
93
94 def get_max_fuyu_image_tokens(ctx: InputContext):
94 - ncol, nrow = get_max_fuyu_image_feature_size()
95 - return (ncol + 1) * nrow
96 -
97 -
98 -def dummy_seq_data_for_fuyu(ctx: InputContext, seq_len: int, num_images: int):
99 - ncol, nrow = get_max_fuyu_image_feature_size()
100 - image_feature_size = get_max_fuyu_image_tokens(ctx)
101 -
102 - image_token_ids = (
103 - array(VLLM_TOKEN_ID_ARRAY_TYPE, [_IMAGE_TOKEN_ID]) * ncol +
104 - array(VLLM_TOKEN_ID_ARRAY_TYPE, [_NEWLINE_TOKEN_ID])) * nrow
105 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, image_token_ids) * num_images
106 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
107 - [0]) * (seq_len - image_feature_size * num_images)
108 - return SequenceData(token_ids), {
109 - "image":
110 - consecutive_placeholder_ranges(num_items=num_images,
111 - item_size=image_feature_size)
112 - }
113 -
114 -
115 -def dummy_image_for_fuyu(
116 - num_images: int,
117 - *,
118 - image_width: int,
119 - image_height: int,
120 -):
121 - image = Image.new("RGB", (image_width, image_height), color=0)
122 - return {"image": image if num_images == 1 else [image] * num_images}
123 -
124 -
125 -def dummy_data_for_fuyu(ctx: InputContext, seq_len: int,
126 - mm_counts: Mapping[str, int]):
127 - num_images = mm_counts["image"]
128 - seq_data, ranges = dummy_seq_data_for_fuyu(ctx, seq_len, num_images)
129 - mm_data = dummy_image_for_fuyu(num_images,
130 - image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
131 - image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT)
132 - return DummyData(seq_data, mm_data, ranges)
133 -
134 -
135 -def _fuyu_image_preprocess(image_processor: FuyuImageProcessor,
136 - data: List[Image.Image]):
137 - image_encoding = image_processor.preprocess(data, return_tensors="pt")
138 - batch_images = torch.stack([img[0] for img in image_encoding["images"]
139 - ]).unsqueeze(1)
140 - image_unpadded_heights = torch.tensor(
141 - image_encoding["image_unpadded_heights"])
142 - image_unpadded_widths = torch.tensor(
143 - image_encoding["image_unpadded_widths"])
144 -
145 - batch_size = len(image_encoding["images"])
146 - image_present = torch.ones(batch_size, 1, 1)
147 - model_image_input = image_processor.preprocess_with_tokenizer_info(
148 - image_input=batch_images,
149 - image_present=image_present,
150 - image_unpadded_h=image_unpadded_heights,
151 - image_unpadded_w=image_unpadded_widths,
152 - image_placeholder_id=_IMAGE_TOKEN_ID,
153 - image_newline_id=_NEWLINE_TOKEN_ID,
154 - variable_sized=True,
95 + ncols, nrows = _get_fuyu_num_image_tokens(
96 + image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
97 + image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
98 )
156 - return model_image_input
157 -
158 -
159 -def input_processor_for_fuyu(ctx: InputContext, inputs: DecoderOnlyInputs):
160 - multi_modal_data = inputs.get("multi_modal_data")
161 - if multi_modal_data is None or "image" not in multi_modal_data:
162 - return inputs
163 -
164 - model_config = ctx.model_config
165 - image_data = multi_modal_data["image"]
166 - new_multi_modal_data = {}
167 - image_list = image_data if isinstance(image_data, list) else [image_data]
168 -
169 - # process image data
170 - if is_list_of(image_list, Image.Image):
171 - # Fuyu's image_processor can also finish token padding
172 - image_processor: FuyuImageProcessor = cached_get_image_processor(
173 - model_config.model)
174 -
175 - model_image_input = _fuyu_image_preprocess(image_processor, image_data)
176 - image_patches = torch.cat([
177 - image_patch[0]
178 - for image_patch in model_image_input["image_patches"]
179 - ])
180 - new_multi_modal_data["image"] = image_patches
181 -
182 - elif is_list_of(image_list, torch.Tensor):
183 - raise NotImplementedError("Embeddings input is not supported yet")
184 - else:
185 - raise TypeError(f"Invalid image type: {type(image_data)}")
186 -
187 - # process prompts
188 - prompt = inputs.get("prompt")
189 - prompt_token_ids = inputs["prompt_token_ids"]
190 - tokenizer = cached_get_tokenizer(model_config.model)
191 - # dim0 is batch_size, dim1 is subseq_size which will always be 1
192 - image_input_ids: List[List[
193 - torch.Tensor]] = model_image_input["image_input_ids"]
194 - image_input_ids = image_input_ids[0][0].tolist()
195 - bos_token = tokenizer.encode("<s>", add_special_tokens=False)[1:]
196 - boa_token = tokenizer.encode("\x04", add_special_tokens=False)[1:]
197 -
198 - new_prompt = prompt + "\x04"
199 - new_prompt_token_ids = image_input_ids + bos_token + prompt_token_ids[
200 - 1:] + boa_token
201 -
202 - return token_inputs(prompt=new_prompt,
203 - prompt_token_ids=new_prompt_token_ids,
204 - multi_modal_data=new_multi_modal_data)
205 -
206 -
207 -def input_mapper_for_fuyu(ctx: InputContext, data: object):
208 - model_config = ctx.model_config
209 - data_list = data if isinstance(data, list) else [data]
210 - if is_list_of(data_list, Image.Image):
211 - # Fuyu's image_processor can also finish token padding
212 - image_processor: FuyuImageProcessor = cached_get_image_processor(
213 - model_config.model)
214 -
215 - model_image_input = _fuyu_image_preprocess(image_processor, data_list)
216 - data = torch.stack([
217 - image_patch[0]
218 - for image_patch in model_image_input["image_patches"]
219 - ])
220 -
221 - # image has been processed with prompt in input processor
222 - return MultiModalKwargs({"pixel_values": data})
223 -
224 -
225 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_fuyu)
99 +
100 + return (ncols + 1) * nrows
101 +
102 +
103 +class FuyuMultiModalProcessor(BaseMultiModalProcessor):
104 +
105 + def _get_hf_processor(self) -> FuyuProcessor:
106 + return self.ctx.get_hf_processor(FuyuProcessor)
107 +
108 + def _call_hf_processor(
109 + self,
110 + prompt: str,
111 + mm_data: Mapping[str, object],
112 + mm_kwargs: Mapping[str, object],
113 + ) -> BatchFeature:
114 +
115 + if not mm_data:
116 + # Avoid warning from HF logger for text-only input
117 + # Input_ids format: bos_token_id + prompt_token_ids + boa_token_id
118 + # Tokenizer won't add boa_token_id by default, we add it manually.
119 + tokenizer = self._get_tokenizer()
120 + boa_token_id: int = tokenizer.vocab["<0x04>"] # type: ignore
121 + prompt_ids = tokenizer.encode(prompt) + [boa_token_id]
122 + return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
123 +
124 + processed_outputs = super()._call_hf_processor(
125 + prompt=prompt,
126 + mm_data=mm_data,
127 + mm_kwargs=mm_kwargs,
128 + )
129 +
130 + image_patches = processed_outputs.get("image_patches")
131 + if image_patches is not None:
132 + images = mm_data["images"]
133 + assert isinstance(images, list)
134 +
135 + # Original output: (1, num_images, Pn, Px * Py * C)
136 + # New output: (num_images, Pn, Px * Py * C)
137 + assert (isinstance(image_patches, list)
138 + and len(image_patches) == 1)
139 + assert (isinstance(image_patches[0], torch.Tensor)
140 + and len(image_patches[0]) == len(images))
141 +
142 + processed_outputs["image_patches"] = image_patches[0]
143 +
144 + return processed_outputs
145 +
146 + def _get_mm_fields_config(
147 + self,
148 + hf_inputs: BatchFeature,
149 + hf_processor_mm_kwargs: Mapping[str, object],
150 + ) -> Mapping[str, MultiModalFieldConfig]:
151 + return dict(image_patches=MultiModalFieldConfig.batched("image"))
152 +
153 + def _get_prompt_replacements(
154 + self,
155 + mm_items: MultiModalDataItems,
156 + hf_processor_mm_kwargs: Mapping[str, object],
157 + out_mm_kwargs: MultiModalKwargs,
158 + ) -> list[PromptReplacement]:
159 + hf_config = self.ctx.get_hf_config(FuyuConfig)
160 + bos_token_id = hf_config.bos_token_id
161 +
162 + tokenizer = self._get_tokenizer()
163 + eot_token_id = tokenizer.bos_token_id
164 + assert isinstance(eot_token_id, int)
165 +
166 + hf_processor = self._get_hf_processor()
167 + image_processor: FuyuImageProcessor = hf_processor.image_processor
168 + target_size = image_processor.size
169 + target_height, target_width = (target_size["height"],
170 + target_size["width"])
171 +
172 + def get_replacement_fuyu(item_idx: int):
173 + images = mm_items.get_items("image", ImageProcessorItems)
174 + image_size = images.get_image_size(item_idx)
175 + width, height = image_size.width, image_size.height
176 + if not (width <= target_width and height <= target_height):
177 + height_scale_factor = target_height / height
178 + width_scale_factor = target_width / width
179 + optimal_scale_factor = min(height_scale_factor,
180 + width_scale_factor)
181 +
182 + height = int(height * optimal_scale_factor)
183 + width = int(width * optimal_scale_factor)
184 +
185 + ncols, nrows = _get_fuyu_num_image_tokens(
186 + image_width=width,
187 + image_height=height,
188 + )
189 +
190 + return (([_IMAGE_TOKEN_ID] * ncols + [_NEWLINE_TOKEN_ID]) * nrows +
191 + [bos_token_id])
192 +
193 + return [
194 + PromptReplacement(
195 + modality="image",
196 + target=[eot_token_id],
197 + replacement=get_replacement_fuyu,
198 + )
199 + ]
200 +
201 + def apply(
202 + self,
203 + prompt_text: str,
204 + mm_data: MultiModalDataDict,
205 + hf_processor_mm_kwargs: Mapping[str, object],
206 + ) -> MultiModalInputsV2:
207 + result = super().apply(prompt_text, mm_data, hf_processor_mm_kwargs)
208 +
209 + # Only |SPEAKER| (image) tokens should be considered as placeholders,
210 + # so we ignore the trailing bos_token_id
211 + result["mm_placeholders"] = {
212 + modality: [
213 + PlaceholderRange(offset=p["offset"], length=p["length"] - 1)
214 + for p in ps
215 + ]
216 + for modality, ps in result["mm_placeholders"].items()
217 + }
218 +
219 + return result
220 +
221 + def _get_dummy_mm_inputs(
222 + self,
223 + mm_counts: Mapping[str, int],
224 + ) -> ProcessorInputs:
225 + num_images = mm_counts.get("image", 0)
226 +
227 + mm_data = {
228 + "image":
229 + self._get_dummy_images(width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
230 + height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
231 + num_images=num_images)
232 + }
233 +
234 + return ProcessorInputs(
235 + prompt_text="",
236 + mm_data=mm_data,
237 + )
238 +
239 +
240 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_fuyu_image_tokens)
227 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_fuyu)
228 -@INPUT_REGISTRY.register_input_processor(input_processor_for_fuyu)
241 +@MULTIMODAL_REGISTRY.register_processor(FuyuMultiModalProcessor)
242 class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
243
244 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
@@ -280,28 +293,32 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
293 return data.to(self.vision_embed_tokens.weight.dtype)
294
295 def _parse_and_validate_image_input(
283 - self, **kwargs: object) -> Optional[FuyuImagePixelInputs]:
284 - pixel_values = kwargs.pop("pixel_values", None)
285 -
286 - if pixel_values is not None:
287 - if not isinstance(pixel_values, (torch.Tensor, list)):
296 + self, **kwargs: object) -> Optional[FuyuImagePatchInputs]:
297 + image_patches = kwargs.pop("image_patches", None)
298 + if image_patches is not None:
299 + if not isinstance(image_patches, (torch.Tensor, list)):
300 raise ValueError("Incorrect type of image patches. "
289 - f"Got type: {type(pixel_values)}")
301 + f"Got type: {type(image_patches)}")
302
291 - return FuyuImagePixelInputs(
292 - type="pixel_values",
303 + image_patches_flat = flatten_bn(image_patches)
304 +
305 + return FuyuImagePatchInputs(
306 + type="image_patches",
307 data=self._validate_pixel_values(
294 - flatten_bn(pixel_values, concat=True)),
308 + flatten_bn(image_patches_flat, concat=True)),
309 + patches_per_image=[x.size(0) for x in image_patches_flat],
310 )
311
312 return None
313
314 def _process_image_input(
300 - self, image_input: FuyuImagePixelInputs) -> torch.Tensor:
315 + self, image_input: FuyuImagePatchInputs) -> NestedTensors:
316 + image_patches = image_input["data"]
317 + patches_per_image = image_input["patches_per_image"]
318
319 assert self.vision_embed_tokens is not None
303 - vision_embeddings, _ = self.vision_embed_tokens(image_input["data"])
304 - return vision_embeddings
320 + vision_embeddings, _ = self.vision_embed_tokens(image_patches)
321 + return vision_embeddings.split(patches_per_image, dim=0)
322
323 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
324 image_input = self._parse_and_validate_image_input(**kwargs)
vllm/model_executor/models/idefics2_vision_model.py
+4 -2
@@ -69,7 +69,8 @@ class Idefics2VisionEmbeddings(nn.Module):
69 patch_attention_mask: torch.BoolTensor,
70 tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
71 batch_size, _, max_im_h, max_im_w = pixel_values.shape
72 - patch_embeds = self.patch_embedding(pixel_values)
72 + target_dtype = self.patch_embedding.weight.dtype
73 + patch_embeds = self.patch_embedding(pixel_values.to(target_dtype))
74 embeddings = patch_embeds.flatten(2).transpose(1, 2)
75 max_nb_patches_h, max_nb_patches_w = (
76 max_im_h // self.patch_size,
@@ -309,7 +310,8 @@ class Idefics2VisionTransformer(nn.Module):
310 hidden_states = self.embeddings(
311 pixel_values=pixel_values,
312 patch_attention_mask=patch_attention_mask,
312 - tgt_sizes=tgt_sizes)
313 + tgt_sizes=tgt_sizes,
314 + )
315 encoder_outputs = self.encoder(hidden_states)
316 last_hidden_state = self.post_layernorm(encoder_outputs)
317 return last_hidden_state
vllm/model_executor/models/llava.py
+2 -2
@@ -144,8 +144,8 @@ class LlavaMultiModalProcessor(BaseMultiModalProcessor):
144 # Original output: (1, num_images, C, H, W)
145 # New output: (num_images, C, H, W)
146 assert (isinstance(pixel_values, list)
147 - and len(pixel_values) == 1
148 - and isinstance(pixel_values[0], list)
147 + and len(pixel_values) == 1)
148 + assert (isinstance(pixel_values[0], list)
149 and len(pixel_values[0]) == len(images))
150
151 processed_outputs["pixel_values"] = pixel_values[0]
vllm/model_executor/models/llava_next.py
+2 -4
@@ -528,10 +528,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
528 stacked_image_features = self._image_pixels_to_features(
529 self.vision_tower, stacked_pixel_values)
530
531 - return [
532 - self.multi_modal_projector(image_features) for image_features in
533 - torch.split(stacked_image_features, num_patches_per_batch)
534 - ]
531 + return torch.split(self.multi_modal_projector(stacked_image_features),
532 + num_patches_per_batch)
533
534 def _process_image_input(
535 self,
vllm/model_executor/models/pixtral.py
+6 -6
@@ -1,8 +1,8 @@
1 +import math
2 from dataclasses import dataclass, fields
3 from functools import cached_property
4 from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union
5
5 -import numpy
6 import torch
7 import torch.nn as nn
8 import torch.nn.functional as F
@@ -306,7 +306,7 @@ class PixtralForConditionalGeneration(nn.Module, SupportsMultiModal,
306 images: Optional[Union[List[List[torch.Tensor]], List[torch.Tensor],
307 torch.Tensor]] = None,
308 image_tokens: Optional[torch.Tensor] = None,
309 - ) -> Optional[List[torch.Tensor]]:
309 + ) -> Tuple[Optional[List[torch.Tensor]], Optional[torch.Tensor]]:
310 if images is None:
311 return None, None
312
@@ -604,11 +604,11 @@ class VisionTransformer(nn.Module):
604 return self.args.image_size // self.args.patch_size
605
606 @property
607 - def device(self) -> torch.device:
607 + def device(self) -> torch.types.Device:
608 return next(self.parameters()).device
609
610 @property
611 - def dtype(self) -> torch.device:
611 + def dtype(self) -> torch.dtype:
612 return next(self.parameters()).dtype
613
614 @property
@@ -741,8 +741,8 @@ def get_pixtral_hf_image_feature_size(hf_config: PixtralVisionConfig,
741 ratio = max(image_width / max_width, image_height / max_height)
742
743 if ratio > 1:
744 - image_width = int(numpy.ceil(image_width / ratio))
745 - image_height = int(numpy.ceil(image_height / ratio))
744 + image_width = int(math.ceil(image_width / ratio))
745 + image_height = int(math.ceil(image_height / ratio))
746
747 num_height_tokens, num_width_tokens = _get_pixtral_hf_num_image_tokens(
748 (image_height, image_width),
vllm/model_executor/models/qwen2_audio.py
+8 -6
@@ -23,7 +23,6 @@ from functools import cached_property
23 from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
24 Union)
25
26 -import numpy as np
26 import torch
27 import torch.nn as nn
28 from transformers import BatchFeature
@@ -177,16 +176,19 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
176 mm_counts: Mapping[str, int],
177 ) -> ProcessorInputs:
178 feature_extractor = self._get_feature_extractor()
179 +
180 sampling_rate = feature_extractor.sampling_rate
181 audio_len = feature_extractor.chunk_length * sampling_rate
182 + num_audios = mm_counts.get("audio", 0)
183
183 - audio_count = mm_counts.get("audio", 0)
184 - audio = np.zeros(audio_len)
185 - data = {"audio": [audio] * audio_count}
184 + mm_data = {
185 + "audio":
186 + self._get_dummy_audios(length=audio_len, num_audios=num_audios)
187 + }
188
189 return ProcessorInputs(
188 - prompt_text="<|AUDIO|>" * audio_count,
189 - mm_data=data,
190 + prompt_text="<|AUDIO|>" * num_audios,
191 + mm_data=mm_data,
192 )
193
194
vllm/model_executor/models/qwen2_vl.py
+9 -8
@@ -29,7 +29,6 @@ import torch
29 import torch.nn as nn
30 import torch.nn.functional as F
31 from einops import rearrange, repeat
32 -from PIL import Image
32 from transformers import BatchFeature
33 from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
34 Qwen2VLProcessor)
@@ -882,12 +881,10 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
881 self,
882 mm_counts: Mapping[str, int],
883 ) -> ProcessorInputs:
885 - num_images = mm_counts.get("image", 0)
884 hf_processor = self._get_hf_processor()
887 - image_token: str = hf_processor.image_token
885 image_processor = _get_image_processor(hf_processor)
886
890 - data = {}
887 + image_token: str = hf_processor.image_token
888 resized_height, resized_width = smart_resize(
889 height=9999999,
890 width=9999999,
@@ -895,14 +892,18 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
892 min_pixels=image_processor.min_pixels,
893 max_pixels=image_processor.max_pixels,
894 )
895 + num_images = mm_counts.get("image", 0)
896
899 - dummy_image = Image.new("RGB", (resized_width, resized_height),
900 - color=0)
901 - data["image"] = [dummy_image] * num_images
897 + mm_data = {
898 + "image":
899 + self._get_dummy_images(width=resized_width,
900 + height=resized_height,
901 + num_images=num_images)
902 + }
903
904 return ProcessorInputs(
905 prompt_text=image_token * num_images,
905 - mm_data=data,
906 + mm_data=mm_data,
907 )
908
909
vllm/model_executor/models/ultravox.py
+8 -5
@@ -188,16 +188,19 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
188 mm_counts: Mapping[str, int],
189 ) -> ProcessorInputs:
190 feature_extractor = self._get_feature_extractor()
191 +
192 sampling_rate = feature_extractor.sampling_rate
193 audio_len = feature_extractor.chunk_length * sampling_rate
194 + num_audios = mm_counts.get("audio", 0)
195
194 - audio_count = mm_counts.get("audio", 0)
195 - audio = np.zeros(audio_len)
196 - data = {"audio": [audio] * audio_count}
196 + mm_data = {
197 + "audio":
198 + self._get_dummy_audios(length=audio_len, num_audios=num_audios)
199 + }
200
201 return ProcessorInputs(
199 - prompt_text="<|audio|>" * audio_count,
200 - mm_data=data,
202 + prompt_text="<|audio|>" * num_audios,
203 + mm_data=mm_data,
204 )
205
206
vllm/multimodal/processing.py
+55 -13
@@ -1,15 +1,17 @@
1 import pickle
2 import re
3 from abc import ABC, abstractmethod
4 +from collections import defaultdict
5 from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
6 from dataclasses import dataclass, field
7 from functools import lru_cache
8 from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
9
10 import numpy as np
11 +import numpy.typing as npt
12 import torch
13 from blake3 import blake3
12 -from PIL.Image import Image
14 +from PIL import Image
15 from transformers import BatchFeature, ProcessorMixin
16
17 from vllm.inputs import DummyData, InputProcessingContext
@@ -353,13 +355,13 @@ def _replace_matches(
355 ) -> list[_S]:
356 out_seqs = list[_S]()
357 prev_end_idx = 0
356 - next_idx_by_modality = {modality: 0 for modality in mm_item_counts}
358 + next_idx_by_modality = defaultdict[str, int](lambda: 0)
359
360 for match in _resolve_matches(prompt, matches):
361 modality = match.modality
362
363 item_idx = next_idx_by_modality[modality]
362 - if item_idx >= mm_item_counts[modality]:
364 + if item_idx >= mm_item_counts.get(modality, 0):
365 continue
366
367 start_idx = match.start_idx
@@ -513,7 +515,7 @@ class ProcessingCache:
515 return obj.encode("utf-8")
516 if isinstance(obj, bytes):
517 return obj
516 - if isinstance(obj, Image):
518 + if isinstance(obj, Image.Image):
519 return obj.tobytes()
520
521 # Convertible to NumPy arrays
@@ -673,10 +675,14 @@ class BaseMultiModalProcessor(ABC):
675 Given the original multi-modal items for this modality
676 and HF-processed data, output the replacements to perform.
677
676 - Note:
677 - Even when the HF processor already performs replacement for us,
678 - we still use this replacement information to determine
679 - the placeholder token positions for each multi-modal item.
678 + Notes:
679 + - You should not assume that HF processor always performs prompt
680 + replacement: in :meth:`_apply_hf_processor_missing`, this method
681 + is called on text-only and multimodal-only inputs separately,
682 + instead of passing them in the same call.
683 + - The replacement information returned by this method is also used
684 + to determine the placeholder token positions for each multi-modal
685 + item.
686 """
687 raise NotImplementedError
688
@@ -710,6 +716,10 @@ class BaseMultiModalProcessor(ABC):
716 mm_data: Mapping[str, object],
717 mm_kwargs: Mapping[str, object],
718 ) -> BatchFeature:
719 + """
720 + Call the HF processor on the prompt text and
721 + associated multi-modal data.
722 + """
723 return self.ctx.call_hf_processor(
724 self._get_hf_processor(**mm_kwargs),
725 dict(text=prompt, **mm_data),
@@ -723,7 +733,8 @@ class BaseMultiModalProcessor(ABC):
733 hf_processor_mm_kwargs: Mapping[str, object],
734 ) -> tuple[list[int], MultiModalKwargs]:
735 """
726 - Apply the HF processor on the full prompt text and multi-modal data.
736 + Wrapper of :meth:`_call_hf_processor` that applies
737 + additional pre-processing and post-processing.
738 """
739 processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
740
@@ -754,10 +765,11 @@ class BaseMultiModalProcessor(ABC):
765 Apply the HF processor on the full prompt text, but only on the
766 multi-modal data that are missing from the cache.
767
757 - Note: We pass prompt text and multi-modal data into the HF processor
758 - in separate calls to avoid HF prompt replacement being done for
759 - cached items; instead, we rely on our own prompt replacement logic
760 - for the full text.
768 + Note:
769 + We pass prompt text and multi-modal data into the HF processor
770 + in separate calls to avoid HF prompt replacement being done for
771 + cached items; instead, we rely on our own prompt replacement logic
772 + (:meth:`_get_prompt_replacements`) for the full text.
773 """
774 mm_missing_counts = mm_missing_data_items.get_all_counts()
775
@@ -1010,6 +1022,36 @@ class BaseMultiModalProcessor(ABC):
1022 mm_placeholders=mm_placeholders,
1023 )
1024
1025 + def _get_dummy_audios(
1026 + self,
1027 + *,
1028 + length: int,
1029 + num_audios: int,
1030 + ) -> list[npt.NDArray]:
1031 + audio = np.zeros((length, ))
1032 + return [audio] * num_audios
1033 +
1034 + def _get_dummy_images(
1035 + self,
1036 + *,
1037 + width: int,
1038 + height: int,
1039 + num_images: int,
1040 + ) -> list[Image.Image]:
1041 + image = Image.new("RGB", (width, height), color=0)
1042 + return [image] * num_images
1043 +
1044 + def _get_dummy_videos(
1045 + self,
1046 + *,
1047 + width: int,
1048 + height: int,
1049 + num_frames: int,
1050 + num_videos: int,
1051 + ) -> list[npt.NDArray]:
1052 + video = np.zeros((num_frames, width, height, 3))
1053 + return [video] * num_videos
1054 +
1055 @abstractmethod
1056 def _get_dummy_mm_inputs(
1057 self,
vllm/multimodal/utils.py
+7 -3
@@ -400,15 +400,19 @@ def repeat_and_pad_placeholder_tokens(
400 placeholder_token_idx = 0
401 for i, token in enumerate(prompt_token_ids):
402 if token == placeholder_token_id:
403 + curr_repeat_count = repeat_count[placeholder_token_idx]
404 replacement_ids = repeat_and_pad_token(
405 placeholder_token_id,
405 - repeat_count=repeat_count[placeholder_token_idx],
406 + repeat_count=curr_repeat_count,
407 pad_token_left=pad_token_left,
408 pad_token_right=pad_token_right,
409 )
410 + offset = len(new_token_ids)
411 + if pad_token_left is not None:
412 + offset += 1
413 placeholder_ranges.append({
410 - "offset": len(new_token_ids),
411 - "length": len(replacement_ids)
414 + "offset": offset,
415 + "length": curr_repeat_count,
416 })
417 new_token_ids.extend(replacement_ids)
418 placeholder_token_idx += 1
vllm/v1/worker/gpu_model_runner.py
+14 -1
@@ -647,10 +647,23 @@ class GPUModelRunner:
647 self.mm_registry.get_max_tokens_per_item_by_modality(
648 self.model_config).values())
649
650 - max_num_mm_items = min(
650 + max_num_mm_items_encoder_budget = min(
651 self.max_num_encoder_input_tokens,
652 self.encoder_cache_size) // max_tokens_per_mm_item
653
654 + max_mm_items_per_req = max(
655 + self.mm_registry.get_mm_limits_per_prompt(
656 + self.model_config).values())
657 +
658 + # NOTE: We do not consider max_num_batched_tokens on purpose
659 + # because the multimodal embeddings can be generated in advance
660 + # and chunked prefilled.
661 + max_num_mm_items_decoder_budget = self.max_num_reqs * \
662 + max_mm_items_per_req
663 +
664 + max_num_mm_items = min(max_num_mm_items_encoder_budget,
665 + max_num_mm_items_decoder_budget)
666 +
667 # Dummy data definition in V0 may contain multiple multimodal items
668 # (e.g, multiple images) for a single request, therefore here we
669 # always replicate first item by max_num_mm_items times since in V1