@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f58a51358
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] Merged multi-modal processor and V1 support for Qwen-VL (#12504)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 29, 2025 at 00:25 UTC 8f58a5135874770ac8429f4772d7f92fe33094e5
4 files changed +387 -477
docs/source/models/supported_models.md
+1 -1
@@ -745,7 +745,7 @@ See [this page](#generative-models) for more information on how to use generativ
745 - `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc.
746 - ✅︎
747 - ✅︎
748 - -
748 + - ✅︎
749 * - `Qwen2AudioForConditionalGeneration`
750 - Qwen2-Audio
751 - T + A<sup>+</sup>
tests/models/multimodal/processing/test_common.py
+32 -32
@@ -16,7 +16,6 @@ from ...registry import HF_EXAMPLE_MODELS
16
17 def _test_processing_correctness(
18 model_id: str,
19 - modalities: dict[str, bool],
19 hit_rate: float,
20 num_batches: int,
21 simplify_rate: float,
@@ -25,11 +24,6 @@ def _test_processing_correctness(
24 model_info.check_available_online(on_fail="skip")
25 model_info.check_transformers_version(on_fail="skip")
26
28 - limit_mm_per_prompt = {
29 - modality: 3 if supports_multi else 1
30 - for modality, supports_multi in modalities.items()
31 - }
32 -
27 model_config = ModelConfig(
28 model_id,
29 task="auto",
@@ -40,18 +34,29 @@ def _test_processing_correctness(
34 dtype="float16",
35 revision=None,
36 hf_overrides=model_info.hf_overrides,
43 - limit_mm_per_prompt=limit_mm_per_prompt,
37 )
38
39 model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
40 factories = MULTIMODAL_REGISTRY._processor_factories[model_cls]
41 ctx = InputProcessingContext(
42 model_config,
50 - tokenizer=cached_get_tokenizer(model_config.tokenizer),
43 + tokenizer=cached_get_tokenizer(
44 + model_config.tokenizer,
45 + trust_remote_code=model_info.trust_remote_code,
46 + ),
47 )
48 # Ensure that it can fit all of the data
49 cache = ProcessingCache(capacity=1 << 30)
50
51 + processing_info = factories.info(ctx)
52 + supported_mm_limits = processing_info.get_supported_mm_limits()
53 + limit_mm_per_prompt = {
54 + modality: 3 if limit is None else limit
55 + for modality, limit in supported_mm_limits.items()
56 + }
57 +
58 + model_config.get_multimodal_config().limit_per_prompt = limit_mm_per_prompt
59 +
60 baseline_processor = factories.build_processor(ctx, cache=None)
61 cached_processor = factories.build_processor(ctx, cache=cache)
62 dummy_inputs = baseline_processor.dummy_inputs
@@ -82,8 +87,8 @@ def _test_processing_correctness(
87 mm_data = {
88 k:
89 [(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
85 - for _ in range(rng.randint(limit_mm_per_prompt[k]))]
86 - for k in modalities
90 + for _ in range(rng.randint(limit))]
91 + for k, limit in limit_mm_per_prompt.items()
92 }
93
94 mm_counts = {k: len(vs) for k, vs in mm_data.items()}
@@ -135,21 +140,22 @@ def _test_processing_correctness(
140
141 # yapf: disable
142 # True if the model supports multiple data items of the modality per request
138 -@pytest.mark.parametrize(("model_id", "modalities"), [
139 - ("rhymes-ai/Aria", {"image": True}),
140 - ("Salesforce/blip2-opt-2.7b", {"image": False}),
141 - ("facebook/chameleon-7b", {"image": False}),
142 - ("deepseek-ai/deepseek-vl2-tiny", {"image": True}),
143 - ("adept/fuyu-8b", {"image": False}),
144 - ("llava-hf/llava-1.5-7b-hf", {"image": True}),
145 - ("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
146 - ("llava-hf/LLaVA-NeXT-Video-7B-hf", {"video": False}),
147 - ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", {"image": True, "video": True}), # noqa: E501
148 - ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
149 - ("mistral-community/pixtral-12b", {"image": True}),
150 - ("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
151 - ("Qwen/Qwen2-Audio-7B-Instruct", {"audio": True}),
152 - ("fixie-ai/ultravox-v0_3", {"audio": True}),
143 +@pytest.mark.parametrize("model_id", [
144 + "rhymes-ai/Aria",
145 + "Salesforce/blip2-opt-2.7b",
146 + "facebook/chameleon-7b",
147 + "deepseek-ai/deepseek-vl2-tiny",
148 + "adept/fuyu-8b",
149 + "llava-hf/llava-1.5-7b-hf",
150 + "llava-hf/llava-v1.6-mistral-7b-hf",
151 + "llava-hf/LLaVA-NeXT-Video-7B-hf",
152 + "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
153 + "TIGER-Lab/Mantis-8B-siglip-llama3",
154 + "mistral-community/pixtral-12b",
155 + "Qwen/Qwen-VL-Chat",
156 + "Qwen/Qwen2-VL-2B-Instruct",
157 + "Qwen/Qwen2-Audio-7B-Instruct",
158 + "fixie-ai/ultravox-v0_3",
159 ])
160 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
161 @pytest.mark.parametrize("num_batches", [32])
@@ -157,14 +163,12 @@ def _test_processing_correctness(
163 # yapf: enable
164 def test_processing_correctness(
165 model_id: str,
160 - modalities: dict[str, bool],
166 hit_rate: float,
167 num_batches: int,
168 simplify_rate: float,
169 ):
170 _test_processing_correctness(
171 model_id,
167 - modalities,
172 hit_rate=hit_rate,
173 num_batches=num_batches,
174 simplify_rate=simplify_rate,
@@ -172,16 +176,13 @@ def test_processing_correctness(
176
177
178 # yapf: disable
175 -@pytest.mark.parametrize(("model_id", "modalities"), [
176 - ("microsoft/Phi-3-vision-128k-instruct", {"image": True}),
177 -])
179 +@pytest.mark.parametrize("model_id", ["microsoft/Phi-3-vision-128k-instruct"])
180 @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
181 @pytest.mark.parametrize("num_batches", [32])
182 @pytest.mark.parametrize("simplify_rate", [1.0])
183 # yapf: enable
184 def test_processing_correctness_phi3v(
185 model_id: str,
184 - modalities: dict[str, bool],
186 hit_rate: float,
187 num_batches: int,
188 simplify_rate: float,
@@ -195,7 +196,6 @@ def test_processing_correctness_phi3v(
196
197 _test_processing_correctness(
198 model_id,
198 - modalities,
199 hit_rate=hit_rate,
200 num_batches=num_batches,
201 simplify_rate=simplify_rate,
tests/models/multimodal/processing/test_qwen.py deleted
-144
@@ -1,144 +0,0 @@
1 -"""Tests for Qwen's multimodal preprocessing kwargs."""
2 -from typing import Dict, List, Union
3 -
4 -import pytest
5 -import torch
6 -from PIL.Image import Image
7 -
8 -from vllm.inputs import InputContext, token_inputs
9 -from vllm.multimodal import MultiModalKwargs
10 -from vllm.multimodal.utils import cached_get_tokenizer
11 -
12 -from ....conftest import IMAGE_ASSETS
13 -from ...utils import build_model_context
14 -
15 -### Multimodal preprocessing tests
16 -SAMPLE_IMAGE = IMAGE_ASSETS[0].pil_image
17 -# These values are specific to Qwen-VL/Chat; we can get these from the model
18 -# config also, but they are hardcoded here to keep the parameterize/fixtures
19 -# easy to read.
20 -IMG_START_ID = 151857
21 -IMG_END_ID = 151858
22 -IMG_PAD_ID = 151859
23 -TOKS_PER_IMG = 256
24 -VIS_ENC_DIM = 4096
25 -IMG_SIZE = 448
26 -
27 -
28 -@pytest.fixture()
29 -def input_mapper_for_qwen():
30 - # Lazy import to avoid initializing CUDA during test collection
31 - from vllm.model_executor.models.qwen import input_mapper_for_qwen
32 - return input_mapper_for_qwen
33 -
34 -
35 -@pytest.fixture()
36 -def input_processor_for_qwen():
37 - # Lazy import to avoid initializing CUDA during test collection
38 - from vllm.model_executor.models.qwen import input_processor_for_qwen
39 - return input_processor_for_qwen
40 -
41 -
42 -@pytest.fixture()
43 -def qwen_vl_context() -> InputContext:
44 - """Get an InputContext for Qwen-VL."""
45 - return build_model_context(model_name="Qwen/Qwen-VL",
46 - trust_remote_code=True)
47 -
48 -
49 -# Happy path tests for single/multi-image scenarios for the multimodal
50 -# input processor and mapper, respectively
51 -@pytest.mark.parametrize("num_images", [1, 2])
52 -def test_input_processor_valid_mm_data(input_processor_for_qwen,
53 - qwen_vl_context: InputContext,
54 - num_images: int):
55 - """Happy cases for image inputs to Qwen's multimodal input processor."""
56 - prompt = "".join(
57 - [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
58 - inputs = token_inputs(
59 - prompt=prompt,
60 - # When processing multimodal data for a multimodal model, the qwen
61 - # input processor will overwrite the provided prompt_token_ids with
62 - # the image prompts
63 - prompt_token_ids=[],
64 - multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
65 - )
66 - proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
67 - assert isinstance(proc_inputs, dict)
68 -
69 - # Each image should have one start / stop and a fixed context of 256
70 - proc_tokens = proc_inputs["prompt_token_ids"]
71 - assert proc_tokens.count(IMG_START_ID) == num_images
72 - assert proc_tokens.count(IMG_END_ID) == num_images
73 - assert proc_tokens.count(IMG_PAD_ID) == num_images * TOKS_PER_IMG
74 -
75 -
76 -@pytest.mark.parametrize(
77 - "img_data,expected_shape",
78 - [
79 - # single / multi-image
80 - (SAMPLE_IMAGE, (1, 3, IMG_SIZE, IMG_SIZE)),
81 - (2 * [SAMPLE_IMAGE], (2, 3, IMG_SIZE, IMG_SIZE)),
82 - # single / multi-image embeddings
83 - (torch.rand(
84 - (TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
85 - (torch.rand(
86 - (1, TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
87 - (torch.rand(
88 - (2, TOKS_PER_IMG, VIS_ENC_DIM)), (2, TOKS_PER_IMG, VIS_ENC_DIM)),
89 - ])
90 -def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
91 - qwen_vl_context: InputContext,
92 - img_data: Union[torch.Tensor, List[Image],
93 - Image],
94 - expected_shape: List[int]):
95 - """Happy cases for image inputs to Qwen's multimodal input mapper."""
96 - mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
97 - # Ensure that we get the appropriately shaped pixel_values
98 - # for images and image embeddings, respectively.
99 - assert isinstance(mapped_img_data, MultiModalKwargs)
100 - assert "pixel_values" in mapped_img_data
101 - assert mapped_img_data["pixel_values"].shape == expected_shape
102 -
103 -
104 -# Sad path tests for the multimodal input processor and mapper, respectively
105 -@pytest.mark.parametrize("mm_data", [
106 - {
107 - "image": torch.rand(5)
108 - },
109 - {
110 - "image": torch.rand((5, 5, 5, 5, 5))
111 - },
112 -])
113 -def test_input_processor_invalid_mm_data(input_processor_for_qwen,
114 - qwen_vl_context: InputContext,
115 - mm_data: Dict[str, torch.Tensor]):
116 - """Test sad cases validated in Qwen's multimodal input processor."""
117 - tokenizer = cached_get_tokenizer(qwen_vl_context.model_config.tokenizer,
118 - trust_remote_code=True)
119 - prompt = "Picture 1: <img></img>\n"
120 - prompt_token_ids = tokenizer.encode(prompt)
121 - inputs = token_inputs(prompt=prompt,
122 - prompt_token_ids=prompt_token_ids,
123 - multi_modal_data=mm_data)
124 - # Should fail since we have too many or too few dimensions for embeddings
125 - with pytest.raises(ValueError):
126 - input_processor_for_qwen(qwen_vl_context, inputs)
127 -
128 -
129 -@pytest.mark.parametrize(
130 - "img_data",
131 - [
132 - # Wrong context length
133 - torch.rand((1, TOKS_PER_IMG + 10, VIS_ENC_DIM)),
134 - # Wrong visual encoder output size
135 - torch.rand((1, TOKS_PER_IMG, VIS_ENC_DIM + 10)),
136 - ])
137 -def test_input_mapper_invalid_mm_data(
138 - input_mapper_for_qwen,
139 - qwen_vl_context: InputContext,
140 - img_data: Union[torch.Tensor, List[Image], Image],
141 -):
142 - """Sad cases validated in Qwen VL's multimodal input mapper."""
143 - with pytest.raises(ValueError):
144 - input_mapper_for_qwen(qwen_vl_context, img_data)
vllm/model_executor/models/qwen.py
+354 -300
@@ -4,26 +4,28 @@
4 # LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
5 """Inference-only QWen model compatible with HuggingFace weights."""
6
7 +import copy
8 import math
9 import re
9 -from functools import partial
10 -from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
11 - Optional, Set, Tuple, TypedDict, Union)
10 +import unicodedata
11 +from functools import lru_cache, partial
12 +from typing import (AbstractSet, Any, Callable, Collection, Dict, Iterable,
13 + List, Literal, Mapping, Optional, Set, Tuple, TypedDict,
14 + Union)
15
13 -import numpy as np
16 import torch
15 -from PIL import Image
17 from torch import nn
18 from torchvision import transforms
19 from torchvision.transforms import InterpolationMode
19 -from transformers import PretrainedConfig
20 +from transformers import (BatchFeature, PretrainedConfig, PreTrainedTokenizer,
21 + TensorType)
22 +from transformers.image_utils import ImageInput
23 +from transformers.tokenization_utils_base import TextInput
24
25 from vllm.attention import Attention, AttentionMetadata
26 from vllm.compilation.decorators import support_torch_compile
27 from vllm.config import CacheConfig, VllmConfig
28 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
26 - InputContext, token_inputs)
29 from vllm.logger import init_logger
30 from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
31 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,15 +44,20 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
44 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45 from vllm.model_executor.models.module_mapping import MultiModelKeys
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
45 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
46 -from vllm.multimodal.utils import cached_get_tokenizer
47 -from vllm.sequence import IntermediateTensors, SequenceData
48 -from vllm.utils import is_list_of
47 +from vllm.multimodal import MULTIMODAL_REGISTRY
48 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
49 + NestedTensors)
50 +from vllm.multimodal.parse import MultiModalDataItems
51 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
52 + BaseProcessingInfo, PromptReplacement,
53 + PromptReplacementDetails)
54 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
55 +from vllm.sequence import IntermediateTensors
56
57 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
58 from .utils import (flatten_bn, is_pp_missing_parameter,
59 make_empty_intermediate_tensors_factory, make_layers,
53 - maybe_prefix)
60 + maybe_prefix, merge_multimodal_embeddings)
61
62 logger = init_logger(__name__)
63
@@ -353,8 +360,10 @@ class VisionTransformer(nn.Module):
360 self.ln_post = norm_layer(output_dim)
361 self.proj = nn.Parameter(
362 (output_dim**-0.5) * torch.randn(output_dim, output_dim))
363 +
364 self.image_start_id = image_start_id
365 self.image_end_id = image_start_id + 1
366 + self.image_pad_id = image_start_id + 2
367
368 def forward(self, x: torch.Tensor) -> torch.Tensor:
369 x = x.to(
@@ -383,21 +392,6 @@ class VisionTransformer(nn.Module):
392
393 return x
394
386 - def get_image_positions(self,
387 - input_ids: torch.Tensor) -> Optional[torch.Tensor]:
388 - """Given the input IDs, extracts start/stop points corresponding to
389 - images.
390 -
391 - args:
392 - Returns:
393 - Optional torch tensor corresponding to start/stop pairs of images.
394 - """
395 - if torch.any(input_ids == self.image_start_id):
396 - bos_pos = torch.where(input_ids == self.image_start_id)
397 - eos_pos = torch.where(input_ids == self.image_end_id)
398 - return torch.stack((bos_pos[0], eos_pos[0]), dim=1)
399 - return None
400 -
395
396 class QWenMLP(nn.Module):
397 """MLP for the language component of the Qwen model, which contains a
@@ -579,9 +573,12 @@ class QWenModel(nn.Module):
573 self.make_empty_intermediate_tensors = (
574 make_empty_intermediate_tensors_factory(
575 ["hidden_states", "residual"], config.hidden_size))
582 - self.visual = VisionTransformer(**config.visual,
583 - quant_config=quant_config) if hasattr(
584 - config, "visual") else None
576 +
577 + if (vision_config := getattr(config, "visual", None)):
578 + self.visual = VisionTransformer(**vision_config,
579 + quant_config=quant_config)
580 + else:
581 + self.visual = None
582
583 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
584 return self.wte(input_ids)
@@ -593,38 +590,13 @@ class QWenModel(nn.Module):
590 kv_caches: List[torch.Tensor],
591 attn_metadata: AttentionMetadata,
592 intermediate_tensors: Optional[IntermediateTensors],
596 - pixel_values: Optional[QwenImageInputs],
593 inputs_embeds: Optional[torch.Tensor] = None,
594 ) -> Union[torch.Tensor, IntermediateTensors]:
599 - img_pos = None
600 - # If pixel / visual embeddings are provided, this is a visual model
601 - if pixel_values is not None and self.visual is not None:
602 - if pixel_values["type"] != "image_embeds":
603 - image_embeds = self.visual(pixel_values["data"])
604 - else:
605 - image_embeds = pixel_values["data"]
606 -
607 - # features should be of shape (# images, 256, hidden_dim)
608 - img_pos = self.visual.get_image_positions(input_ids)
609 - if isinstance(
610 - img_pos,
611 - np.ndarray) and img_pos.shape[0] != image_embeds.shape[0]:
612 - raise ValueError(
613 - f"Number of placeholders: {img_pos.shape[0]} "
614 - f"does not match number of images {image_embeds.shape[0]}."
615 - )
616 -
595 if get_pp_group().is_first_rank:
596 if inputs_embeds is not None:
597 hidden_states = inputs_embeds
598 else:
599 hidden_states = self.get_input_embeddings(input_ids)
622 - hidden_states = self.wte(input_ids)
623 - # Merge the image embeddings into the hidden states if actually have
624 - # visual features and the corresponding image tokens
625 - if img_pos is not None:
626 - for idx, (img_bos, img_eos) in enumerate(img_pos):
627 - hidden_states[img_bos + 1:img_eos] = image_embeds[idx]
600 residual = None
601 else:
602 assert intermediate_tensors is not None
@@ -648,159 +620,9 @@ class QWenModel(nn.Module):
620 return hidden_states
621
622
651 -def get_image_text(image_num: int, padding: bool) -> str:
652 - """Retrieves a placeholder text that when tokenized, will be expanded with
653 - image pads.
654 -
655 - Args:
656 - image_num: The number of the image that we want a text prompt for.
657 - Images should be indexed starting at 1.
658 - padding: Whether or not padding should be manually added.
659 -
660 - Returns:
661 - Text placeholder prompt for the image being considered.
662 - """
663 - image_start = f"Picture {image_num}: {IMG_START}"
664 - image_end = f"{IMG_END}\n"
665 - if not padding:
666 - return f"{image_start}{image_end}"
667 - return f"{image_start}{MAX_QWEN_IMG_TOKENS * IMG_PAD}{image_end}"
668 -
669 -
670 -def input_processor_for_qwen(ctx: InputContext,
671 - inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
672 - """Processes the inputs, which may or may not be multimodal.
673 - Multimodal inputs will only be processed if the model has a "visual"
674 - component in its model config, otherwise they'll be ignored.
675 -
676 - Args:
677 - ctx: Context of the loaded model.
678 - inputs: LLM inputs which may have a multi_modal_data attribute.
679 -
680 - Returns:
681 - If the model is language only or not multimodal inputs were provided,
682 - returns inputs unmodified. Otherwise, processes the multimodal
683 - images / image embeddings and adds the fixed-length image placeholders.
684 - """
685 - multi_modal_data = inputs.get("multi_modal_data")
686 -
687 - # Only process images if we have multimodal data and a visual config
688 - hf_config = ctx.get_hf_config()
689 - if (multi_modal_data is None or "image" not in multi_modal_data
690 - or not hasattr(hf_config, "visual")):
691 - return inputs
692 -
693 - prompt = inputs.get("prompt")
694 - prompt_token_ids = inputs["prompt_token_ids"]
695 - model_config = ctx.model_config
696 - tokenizer = cached_get_tokenizer(
697 - model_config.tokenizer,
698 - trust_remote_code=model_config.trust_remote_code)
699 - image_data = multi_modal_data["image"]
700 - if isinstance(image_data, torch.Tensor):
701 - num_dims = len(image_data.shape)
702 - if num_dims < 2 or num_dims > 3:
703 - raise ValueError(
704 - f"Expected img embeds to be have 3 dimensions, got {num_dims}")
705 - num_images = 1 if num_dims == 2 else image_data.shape[0]
706 - elif isinstance(image_data, Image.Image):
707 - num_images = 1
708 - elif is_list_of(image_data, Image.Image):
709 - num_images = len(image_data)
710 - else:
711 - raise TypeError(f"Invalid image type: {type(image_data)}")
712 -
713 - if prompt is None:
714 - prompt = tokenizer.decode(prompt_token_ids)
715 -
716 - # Drops anything between <img>/</img> tags; encoding with the tokenizer
717 - # will automatically add the image pads for the context.
718 - new_prompt, num_matched_images = re.subn(
719 - r"(Picture \d*: <img>).*?(<\/img>\n)",
720 - r"\1\2",
721 - prompt,
722 - )
723 -
724 - if num_matched_images != num_images:
725 - logger.warning(
726 - "Number of matched image placeholders %s doesn't match the number "
727 - "of expected images %s; check your placeholder formatting.",
728 - num_matched_images, num_images)
729 -
730 - new_prompt_token_ids = tokenizer.encode(new_prompt)
731 -
732 - return token_inputs(prompt=new_prompt,
733 - prompt_token_ids=new_prompt_token_ids,
734 - multi_modal_data=multi_modal_data)
735 -
736 -
737 -def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalKwargs:
738 - """Maps the input data to its MultiModalKwargs (if any).
739 -
740 - Args:
741 - ctx: Context of the loaded model.
742 - data: data potentially containing image/image embeddings to be mapped
743 - to pixel_values in .forward() for a visual QWenLMHeadModel model.
744 -
745 - Returns:
746 - MultiModalKwargs containing the stacked normalized images tensor or
747 - image embeddings.
748 - """
749 - # Early exit if we have provided an image to a language only Qwen model
750 - hf_config = ctx.get_hf_config()
751 - if not hasattr(hf_config, "visual"):
752 - logger.warning(
753 - "Images were provided but this model has no visual config; "
754 - "multimodal inputs will not be forwarded to the model.")
755 - return MultiModalKwargs()
756 -
757 - model_config = ctx.model_config
758 - tokenizer = cached_get_tokenizer(
759 - model_config.tokenizer,
760 - trust_remote_code=model_config.trust_remote_code)
761 -
762 - image_pair_tok = tokenizer.encode(IMG_START + IMG_END,
763 - add_special_tokens=False,
764 - return_tensors="pt").squeeze()
765 - image_start_id = image_pair_tok[0]
766 - image_end_id = image_pair_tok[-1]
767 - if (image_start_id + 1) != image_end_id:
768 - raise ValueError(
769 - f"Found image end ID {image_end_id}, but expected {IMG_START} + 1")
770 - if len(image_pair_tok) != (MAX_QWEN_IMG_TOKENS + 2):
771 - raise ValueError(
772 - f"Expected image context length of {MAX_QWEN_IMG_TOKENS}, "
773 - f"but got {image_pair_tok - 2}")
774 -
775 - hf_config = ctx.get_hf_config()
776 - image_size = hf_config.visual["image_size"]
777 - img_emb_size = hf_config.visual["output_dim"]
778 -
779 - if isinstance(data, torch.Tensor):
780 - # It's expected that our values have already been processed
781 - # by the visual transformer; shape is expected to be:
782 - # (# images, 256, hidden_size)
783 - if len(data.shape) == 2:
784 - # Assume only one image embed was provided; unsqueeze the extra dim
785 - data = data.unsqueeze(0)
786 - if len(data.shape) != 3 or data.shape[
787 - 1] != MAX_QWEN_IMG_TOKENS or data.shape[2] != img_emb_size:
788 - raise ValueError(
789 - "Expected image embeds to be a tensor of shape"
790 - f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but "
791 - f"received shape [{data.shape}]")
792 - pixel_values = data
793 - else:
794 - transform = build_normalization_transform(image_size)
795 - if not isinstance(data, (list, tuple)):
796 - data = [data]
797 - transformed_images = [transform(datum) for datum in data]
798 - pixel_values = torch.stack(transformed_images, dim=0)
799 - return MultiModalKwargs({"pixel_values": pixel_values})
800 -
801 -
623 def build_normalization_transform(image_size: int) -> transforms.Compose:
803 - """Builds a normalization transform which can be applied to one or
624 + """
625 + Build a normalization transform which can be applied to one or
626 more input images from which we want to extract visual features.
627
628 Args:
@@ -817,62 +639,251 @@ def build_normalization_transform(image_size: int) -> transforms.Compose:
639 ])
640
641
820 -def dummy_data_for_qwen(
821 - ctx: InputContext,
822 - seq_len: int,
823 - mm_counts: Mapping[str, int],
824 -) -> DummyData:
825 - """Build dummy data for warming up Qwen models; this will only contain text
826 - matching the defaults for VLLM unless the model has a visual config.
642 +@lru_cache(maxsize=1)
643 +def _get_tokenizer_without_image_pad(
644 + tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
645 + """
646 + The logic of adding image pad tokens should only be applied in
647 + :class:`QWenVLProcessor`, so they are patched out here.
648
828 - Args:
829 - ctx: Context of the loaded model.
830 - seq_len: Number of tokens in the text sequence.
831 - mm_counts: multimodal data counts.
832 -
833 - Returns:
834 - Tuple containing sequential and multimodal data.
649 + The definition of the wrapped tokenizer can be found here:
650 + https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
651 + """
652 + new_tokenizer = copy.deepcopy(tokenizer)
653 +
654 + class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
655 +
656 + def tokenize(
657 + self,
658 + text: str,
659 + allowed_special: Union[AbstractSet[str], str] = "all",
660 + disallowed_special: Union[Collection[str], str] = (),
661 + **kwargs,
662 + ) -> list[Union[bytes, str]]:
663 + text = unicodedata.normalize("NFC", text)
664 +
665 + return [
666 + self.decoder[t] for t in self.tokenizer.encode(
667 + text,
668 + allowed_special=allowed_special,
669 + disallowed_special=disallowed_special,
670 + )
671 + ]
672 +
673 + def _decode(
674 + self,
675 + token_ids: Union[int, List[int]],
676 + skip_special_tokens: bool = False,
677 + errors: Optional[str] = None,
678 + **kwargs,
679 + ) -> str:
680 + if isinstance(token_ids, int):
681 + token_ids = [token_ids]
682 +
683 + return self.tokenizer.decode(
684 + token_ids,
685 + errors=errors or self.errors,
686 + )
687 +
688 + TokenizerWithoutImagePad.__name__ = \
689 + f"{tokenizer.__class__.__name__}WithoutImagePad"
690 +
691 + new_tokenizer.__class__ = TokenizerWithoutImagePad
692 + return new_tokenizer
693 +
694 +
695 +class QWenVLProcessor:
696 + """
697 + This model doesn't define its own HF processor,
698 + so we implement our own one here.
699 +
700 + We call the wrapped tokenizer to automatically insert image pad tokens:
701 + https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
702 +
703 + The image processor is defined here:
704 + https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
705 """
836 - hf_config = ctx.get_hf_config()
837 -
838 - # The presence of a visual config indicates this is a multimodal model.
839 - # If we don't have it, the model is considered an LLM for warmup purposes.
840 - if not hasattr(hf_config, "visual"):
841 - seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
842 - mm_data = None
843 - return DummyData(seq_data, mm_data)
844 -
845 - # We have a visual component - use images to warm up
846 - num_images = mm_counts["image"]
847 - model_config = ctx.model_config
848 - tokenizer = cached_get_tokenizer(
849 - model_config.tokenizer,
850 - trust_remote_code=model_config.trust_remote_code)
851 -
852 - # Build the image prompts with no imgpads; the tokenizer will add img pads
853 - image_prompt = ''.join(
854 - [get_image_text(idx, False) for idx in range(1, num_images + 1)])
855 - toks = tokenizer.encode(image_prompt, add_special_tokens=False)
856 -
857 - # Make sure we actually get the fixed context size per tok padding
858 - num_pads = toks.count(tokenizer.encode(IMG_PAD)[0])
859 - if num_pads != (num_images * MAX_QWEN_IMG_TOKENS):
860 - raise ValueError(
861 - f"Tokenized dummy data should encode {MAX_QWEN_IMG_TOKENS} pads"
862 - f" per image, but got {num_pads} pads for {num_images} image(s)"
863 - " in total. Are you using a qwen tokenizer?")
864 -
865 - # Ensure the number of tokens is at minimum the sequence length provided
866 - if len(toks) < seq_len:
867 - toks += [0] * (seq_len - len(toks))
868 -
869 - seq_data = SequenceData.from_seqs(toks)
870 -
871 - # Build the input images; width/height doesn't actually matter here since
872 - # the data will get resized and the # of tokens per image is constant
873 - image = Image.new("RGB", (224, 224), color=0)
874 - mm_data = {"image": image if num_images == 1 else [image] * num_images}
875 - return DummyData(seq_data, mm_data)
706 +
707 + def __init__(
708 + self,
709 + config: PretrainedConfig,
710 + tokenizer: PreTrainedTokenizer,
711 + ) -> None:
712 + super().__init__()
713 +
714 + self.config = config
715 + self.tokenizer = tokenizer
716 +
717 + if hasattr(self.config, "visual"):
718 + self.image_transform = build_normalization_transform(
719 + config.visual["image_size"])
720 + else:
721 + self.image_transform = None
722 +
723 + special_tokens: dict[str,
724 + int] = tokenizer.special_tokens # type: ignore
725 + self.img_start_id = special_tokens[IMG_START]
726 + self.img_end_id = special_tokens[IMG_END]
727 +
728 + def __call__(
729 + self,
730 + text: Optional[Union[TextInput, list[TextInput]]] = None,
731 + images: Optional[Union[ImageInput, list[ImageInput]]] = None,
732 + return_tensors: Optional[Union[str, TensorType]] = None,
733 + ) -> BatchFeature:
734 + if text is None:
735 + text = []
736 + if not isinstance(text, list):
737 + text = [text]
738 + if images is None:
739 + images = []
740 + if not isinstance(images, list):
741 + images = [images]
742 +
743 + text_inputs = self.tokenizer(text)
744 +
745 + if len(images) == 0:
746 + image_inputs = {}
747 + else:
748 + if self.image_transform is None:
749 + raise ValueError("This model does not support image inputs")
750 +
751 + pixel_values = [self.image_transform(image) for image in images]
752 + image_inputs = {"pixel_values": torch.stack(pixel_values)}
753 +
754 + return BatchFeature(
755 + {
756 + **text_inputs,
757 + **image_inputs,
758 + },
759 + tensor_type=return_tensors,
760 + )
761 +
762 +
763 +class QWenVLProcessingInfo(BaseProcessingInfo):
764 +
765 + def get_tokenizer(self) -> PreTrainedTokenizer:
766 + tokenizer = self.ctx.tokenizer
767 + assert isinstance(tokenizer, PreTrainedTokenizer)
768 +
769 + return _get_tokenizer_without_image_pad(tokenizer)
770 +
771 + def get_hf_processor(self) -> QWenVLProcessor:
772 + tokenizer = self.ctx.tokenizer
773 + assert isinstance(tokenizer, PreTrainedTokenizer)
774 +
775 + return QWenVLProcessor(self.get_hf_config(), tokenizer)
776 +
777 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
778 + return {"image": None}
779 +
780 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
781 + return {"image": self.get_num_image_tokens()}
782 +
783 + def get_num_image_tokens(self) -> int:
784 + return MAX_QWEN_IMG_TOKENS
785 +
786 +
787 +class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
788 +
789 + def get_dummy_processor_inputs(
790 + self,
791 + seq_len: int,
792 + mm_counts: Mapping[str, int],
793 + ) -> ProcessorInputs:
794 + hf_config = self.info.get_hf_config()
795 + if not hasattr(hf_config, "visual"):
796 + return ProcessorInputs(prompt_text="", mm_data={})
797 +
798 + vision_config = hf_config.visual
799 +
800 + max_image_size = vision_config["image_size"]
801 + num_images = mm_counts.get("image", 0)
802 +
803 + mm_data = {
804 + "image":
805 + self._get_dummy_images(width=max_image_size,
806 + height=max_image_size,
807 + num_images=num_images)
808 + }
809 +
810 + return ProcessorInputs(
811 + prompt_text="".join(f"Picture {i}: {IMG_START}{IMG_END}\n"
812 + for i in range(1, num_images + 1)),
813 + mm_data=mm_data,
814 + )
815 +
816 +
817 +class QWenVLMultiModalProcessor(BaseMultiModalProcessor[QWenVLProcessingInfo]):
818 +
819 + def _call_hf_processor(
820 + self,
821 + prompt: str,
822 + mm_data: Mapping[str, object],
823 + mm_kwargs: Mapping[str, object],
824 + ) -> BatchFeature:
825 + # Drops anything between <img>/</img> tags; encoding with the tokenizer
826 + # will automatically add the image pads for the context.
827 + prompt, num_matched_images = re.subn(
828 + r"(Picture \d*: <img>).*?(<\/img>\n)",
829 + r"\1\2",
830 + prompt,
831 + )
832 +
833 + image_data = mm_data.get("images")
834 + if image_data is not None:
835 + assert isinstance(image_data, list)
836 +
837 + num_images = len(image_data)
838 + if num_matched_images != num_images:
839 + logger.warning(
840 + "Number of matched image placeholders %s doesn't match "
841 + "the number of expected images %s; check your placeholder "
842 + "formatting.", num_matched_images, num_images)
843 +
844 + return super()._call_hf_processor(
845 + prompt=prompt,
846 + mm_data=mm_data,
847 + mm_kwargs=mm_kwargs,
848 + )
849 +
850 + def _get_mm_fields_config(
851 + self,
852 + hf_inputs: BatchFeature,
853 + hf_processor_mm_kwargs: Mapping[str, object],
854 + ) -> Mapping[str, MultiModalFieldConfig]:
855 + return dict(
856 + pixel_values=MultiModalFieldConfig.batched("image"),
857 + image_embeds=MultiModalFieldConfig.batched("image"),
858 + )
859 +
860 + def _get_prompt_replacements(
861 + self,
862 + mm_items: MultiModalDataItems,
863 + hf_processor_mm_kwargs: Mapping[str, object],
864 + out_mm_kwargs: MultiModalKwargs,
865 + ) -> list[PromptReplacement]:
866 + tokenizer = self.info.get_tokenizer()
867 + special_tokens: dict[str,
868 + int] = tokenizer.special_tokens # type: ignore
869 +
870 + img_start_id = special_tokens[IMG_START]
871 + img_end_id = special_tokens[IMG_END]
872 + img_pad_id = special_tokens[IMG_PAD]
873 +
874 + num_image_tokens = self.info.get_num_image_tokens()
875 + image_tokens = [img_pad_id] * num_image_tokens
876 +
877 + return [
878 + PromptReplacement(
879 + modality="image",
880 + target=[img_start_id, img_end_id],
881 + replacement=PromptReplacementDetails(
882 + full=[img_start_id] + image_tokens + [img_end_id],
883 + features=image_tokens,
884 + ),
885 + )
886 + ]
887
888
889 class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
@@ -898,38 +909,77 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
909 self.make_empty_intermediate_tensors = (
910 self.transformer.make_empty_intermediate_tensors)
911
901 - def _get_image_input_type(
902 - self,
903 - pixel_values: Optional[torch.Tensor]) -> Optional[QwenImageInputs]:
904 - """Determines if the provided pixel_values are normalized pixel values
905 - or image embeddings.
912 + def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
913 + h = w = self.config.visual["image_size"]
914 + expected_dims = (3, h, w)
915 + actual_dims = tuple(data.shape[1:])
916
907 - Args:
908 - pixel_values: Optional data to processed into visual embeddings.
917 + if actual_dims != expected_dims:
918 + expected_expr = ("batch_size", *map(str, expected_dims))
919 + raise ValueError(
920 + f"The expected shape of pixel values is {expected_expr}. "
921 + f"You supplied {tuple(data.shape)}.")
922 +
923 + return data
924 +
925 + def _parse_and_validate_image_input(
926 + self, **kwargs: object) -> Optional[QwenImageInputs]:
927 + pixel_values = kwargs.pop("pixel_values", None)
928 + image_embeds = kwargs.pop("image_embeds", None)
929 +
930 + if pixel_values is not None:
931 + if not isinstance(pixel_values, torch.Tensor):
932 + raise ValueError("Incorrect type of pixel values. "
933 + f"Got type: {type(pixel_values)}")
934 +
935 + return QwenImagePixelInputs(
936 + type="pixel_values",
937 + data=self._validate_pixel_values(
938 + flatten_bn(pixel_values, concat=True)),
939 + )
940 +
941 + if image_embeds is not None:
942 + if not isinstance(image_embeds, torch.Tensor):
943 + raise ValueError("Incorrect type of image embeddings. "
944 + f"Got type: {type(image_embeds)}")
945 +
946 + return QwenImageEmbeddingInputs(
947 + type="image_embeds",
948 + data=flatten_bn(image_embeds),
949 + )
950
910 - Returns:
911 - None of the QwenImageInputs type used to determine whether or not
912 - the visual transformer needs to process the pixel_values.
913 - """
914 - if pixel_values is not None and self.transformer.visual is not None:
915 - pixel_values = flatten_bn(pixel_values)
916 - if len(pixel_values.shape) == 3 and pixel_values.shape[
917 - 1] == MAX_QWEN_IMG_TOKENS and pixel_values.shape[
918 - 2] == self.config.visual["output_dim"]:
919 - return QwenImageEmbeddingInputs(
920 - type="image_embeds",
921 - data=pixel_values,
922 - )
923 - else:
924 - # If we have the wrong shape, assume we still need to process
925 - return QwenImagePixelInputs(
926 - type="pixel_values",
927 - data=pixel_values,
928 - )
951 return None
952
931 - def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
932 - return self.transformer.get_input_embeddings(input_ids)
953 + def _process_image_input(self,
954 + image_input: QwenImageInputs) -> torch.Tensor:
955 + if image_input["type"] == "image_embeds":
956 + return image_input["data"]
957 +
958 + assert self.transformer.visual is not None
959 + return self.transformer.visual(image_input["data"])
960 +
961 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
962 + image_input = self._parse_and_validate_image_input(**kwargs)
963 + if image_input is None:
964 + return None
965 +
966 + vision_embeddings = self._process_image_input(image_input)
967 + return vision_embeddings
968 +
969 + def get_input_embeddings(
970 + self,
971 + input_ids: torch.Tensor,
972 + multimodal_embeddings: Optional[NestedTensors] = None,
973 + ) -> torch.Tensor:
974 + inputs_embeds = self.transformer.get_input_embeddings(input_ids)
975 +
976 + if multimodal_embeddings is not None:
977 + assert self.transformer.visual is not None
978 + inputs_embeds = merge_multimodal_embeddings(
979 + input_ids, inputs_embeds, multimodal_embeddings,
980 + self.transformer.visual.image_pad_id)
981 +
982 + return inputs_embeds
983
984 def forward(
985 self,
@@ -938,18 +988,23 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
988 kv_caches: List[torch.Tensor],
989 attn_metadata: AttentionMetadata,
990 intermediate_tensors: Optional[IntermediateTensors] = None,
941 - pixel_values: Optional[torch.Tensor] = None,
991 inputs_embeds: Optional[torch.Tensor] = None,
992 + **kwargs: object,
993 ) -> Union[torch.Tensor, IntermediateTensors]:
994 if intermediate_tensors is not None:
995 + inputs_embeds = None
996 +
997 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
998 + # condition is for v0 compatibility.
999 + elif inputs_embeds is None:
1000 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1001 + inputs_embeds = self.get_input_embeddings(input_ids,
1002 + vision_embeddings)
1003 input_ids = None
946 - pixel_values = None
947 - else:
948 - pixel_values = self._get_image_input_type(pixel_values)
1004
1005 hidden_states = self.transformer(input_ids, positions, kv_caches,
1006 attn_metadata, intermediate_tensors,
952 - pixel_values, inputs_embeds)
1007 + inputs_embeds)
1008 return hidden_states
1009
1010 def compute_logits(
@@ -1063,10 +1118,9 @@ class QWenVL(QWenBaseModel, SupportsMultiModal):
1118 tower_model="transformer.visual.transformer")
1119
1120
1066 -@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
1067 -@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS)
1068 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen)
1069 -@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen)
1121 +@MULTIMODAL_REGISTRY.register_processor(QWenVLMultiModalProcessor,
1122 + info=QWenVLProcessingInfo,
1123 + dummy_inputs=QWenVLDummyInputsBuilder)
1124 class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1125 """
1126 QWenLMHeadModel is not only applicable to LLM but also to VL, which is not
@@ -1084,7 +1138,7 @@ class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1138 cls,
1139 vllm_config: VllmConfig,
1140 prefix: str = "",
1087 - ) -> None:
1141 + ) -> QWenBaseModel:
1142 config = vllm_config.model_config.hf_config
1143 # Initialize VL
1144 if hasattr(config, "visual"):