A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] Merged multi-modal processor and V1 support for Qwen-VL (#12504)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 29, 2025 at 00:25 UTC
8f58a5135874770ac8429f4772d7f92fe33094e5
4 files changed
+387
-477
docs/source/models/supported_models.md
+1
-1
@@ -745,7 +745,7 @@ See [this page](#generative-models) for more information on how to use generativ
745
- `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc.
746
- ✅︎
747
- ✅︎
748
- -
748
+ - ✅︎
749
* - `Qwen2AudioForConditionalGeneration`
750
- Qwen2-Audio
751
- T + A<sup>+</sup>
tests/models/multimodal/processing/test_common.py
+32
-32
@@ -16,7 +16,6 @@ from ...registry import HF_EXAMPLE_MODELS
16
17
def _test_processing_correctness(
18
model_id: str,
19
- modalities: dict[str, bool],
19
hit_rate: float,
20
num_batches: int,
21
simplify_rate: float,
@@ -25,11 +24,6 @@ def _test_processing_correctness(
24
model_info.check_available_online(on_fail="skip")
25
model_info.check_transformers_version(on_fail="skip")
26
28
- limit_mm_per_prompt = {
29
- modality: 3 if supports_multi else 1
30
- for modality, supports_multi in modalities.items()
31
- }
32
-
27
model_config = ModelConfig(
28
model_id,
29
task="auto",
@@ -40,18 +34,29 @@ def _test_processing_correctness(
34
dtype="float16",
35
revision=None,
36
hf_overrides=model_info.hf_overrides,
43
- limit_mm_per_prompt=limit_mm_per_prompt,
37
)
38
39
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
40
factories = MULTIMODAL_REGISTRY._processor_factories[model_cls]
41
ctx = InputProcessingContext(
42
model_config,
50
- tokenizer=cached_get_tokenizer(model_config.tokenizer),
43
+ tokenizer=cached_get_tokenizer(
44
+ model_config.tokenizer,
45
+ trust_remote_code=model_info.trust_remote_code,
46
+ ),
47
)
48
# Ensure that it can fit all of the data
49
cache = ProcessingCache(capacity=1 << 30)
50
51
+ processing_info = factories.info(ctx)
52
+ supported_mm_limits = processing_info.get_supported_mm_limits()
53
+ limit_mm_per_prompt = {
54
+ modality: 3 if limit is None else limit
55
+ for modality, limit in supported_mm_limits.items()
56
+ }
57
+
58
+ model_config.get_multimodal_config().limit_per_prompt = limit_mm_per_prompt
59
+
60
baseline_processor = factories.build_processor(ctx, cache=None)
61
cached_processor = factories.build_processor(ctx, cache=cache)
62
dummy_inputs = baseline_processor.dummy_inputs
@@ -82,8 +87,8 @@ def _test_processing_correctness(
87
mm_data = {
88
k:
89
[(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
85
- for _ in range(rng.randint(limit_mm_per_prompt[k]))]
86
- for k in modalities
90
+ for _ in range(rng.randint(limit))]
91
+ for k, limit in limit_mm_per_prompt.items()
92
}
93
94
mm_counts = {k: len(vs) for k, vs in mm_data.items()}
@@ -135,21 +140,22 @@ def _test_processing_correctness(
140
141
# yapf: disable
142
# True if the model supports multiple data items of the modality per request
138
-@pytest.mark.parametrize(("model_id", "modalities"), [
139
- ("rhymes-ai/Aria", {"image": True}),
140
- ("Salesforce/blip2-opt-2.7b", {"image": False}),
141
- ("facebook/chameleon-7b", {"image": False}),
142
- ("deepseek-ai/deepseek-vl2-tiny", {"image": True}),
143
- ("adept/fuyu-8b", {"image": False}),
144
- ("llava-hf/llava-1.5-7b-hf", {"image": True}),
145
- ("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
146
- ("llava-hf/LLaVA-NeXT-Video-7B-hf", {"video": False}),
147
- ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", {"image": True, "video": True}), # noqa: E501
148
- ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
149
- ("mistral-community/pixtral-12b", {"image": True}),
150
- ("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
151
- ("Qwen/Qwen2-Audio-7B-Instruct", {"audio": True}),
152
- ("fixie-ai/ultravox-v0_3", {"audio": True}),
143
+@pytest.mark.parametrize("model_id", [
144
+ "rhymes-ai/Aria",
145
+ "Salesforce/blip2-opt-2.7b",
146
+ "facebook/chameleon-7b",
147
+ "deepseek-ai/deepseek-vl2-tiny",
148
+ "adept/fuyu-8b",
149
+ "llava-hf/llava-1.5-7b-hf",
150
+ "llava-hf/llava-v1.6-mistral-7b-hf",
151
+ "llava-hf/LLaVA-NeXT-Video-7B-hf",
152
+ "llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
153
+ "TIGER-Lab/Mantis-8B-siglip-llama3",
154
+ "mistral-community/pixtral-12b",
155
+ "Qwen/Qwen-VL-Chat",
156
+ "Qwen/Qwen2-VL-2B-Instruct",
157
+ "Qwen/Qwen2-Audio-7B-Instruct",
158
+ "fixie-ai/ultravox-v0_3",
159
])
160
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
161
@pytest.mark.parametrize("num_batches", [32])
@@ -157,14 +163,12 @@ def _test_processing_correctness(
163
# yapf: enable
164
def test_processing_correctness(
165
model_id: str,
160
- modalities: dict[str, bool],
166
hit_rate: float,
167
num_batches: int,
168
simplify_rate: float,
169
):
170
_test_processing_correctness(
171
model_id,
167
- modalities,
172
hit_rate=hit_rate,
173
num_batches=num_batches,
174
simplify_rate=simplify_rate,
@@ -172,16 +176,13 @@ def test_processing_correctness(
176
177
178
# yapf: disable
175
-@pytest.mark.parametrize(("model_id", "modalities"), [
176
- ("microsoft/Phi-3-vision-128k-instruct", {"image": True}),
177
-])
179
+@pytest.mark.parametrize("model_id", ["microsoft/Phi-3-vision-128k-instruct"])
180
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
181
@pytest.mark.parametrize("num_batches", [32])
182
@pytest.mark.parametrize("simplify_rate", [1.0])
183
# yapf: enable
184
def test_processing_correctness_phi3v(
185
model_id: str,
184
- modalities: dict[str, bool],
186
hit_rate: float,
187
num_batches: int,
188
simplify_rate: float,
@@ -195,7 +196,6 @@ def test_processing_correctness_phi3v(
196
197
_test_processing_correctness(
198
model_id,
198
- modalities,
199
hit_rate=hit_rate,
200
num_batches=num_batches,
201
simplify_rate=simplify_rate,
tests/models/multimodal/processing/test_qwen.py
deleted
-144
@@ -1,144 +0,0 @@
1
-"""Tests for Qwen's multimodal preprocessing kwargs."""
2
-from typing import Dict, List, Union
3
-
4
-import pytest
5
-import torch
6
-from PIL.Image import Image
7
-
8
-from vllm.inputs import InputContext, token_inputs
9
-from vllm.multimodal import MultiModalKwargs
10
-from vllm.multimodal.utils import cached_get_tokenizer
11
-
12
-from ....conftest import IMAGE_ASSETS
13
-from ...utils import build_model_context
14
-
15
-### Multimodal preprocessing tests
16
-SAMPLE_IMAGE = IMAGE_ASSETS[0].pil_image
17
-# These values are specific to Qwen-VL/Chat; we can get these from the model
18
-# config also, but they are hardcoded here to keep the parameterize/fixtures
19
-# easy to read.
20
-IMG_START_ID = 151857
21
-IMG_END_ID = 151858
22
-IMG_PAD_ID = 151859
23
-TOKS_PER_IMG = 256
24
-VIS_ENC_DIM = 4096
25
-IMG_SIZE = 448
26
-
27
-
28
-@pytest.fixture()
29
-def input_mapper_for_qwen():
30
- # Lazy import to avoid initializing CUDA during test collection
31
- from vllm.model_executor.models.qwen import input_mapper_for_qwen
32
- return input_mapper_for_qwen
33
-
34
-
35
-@pytest.fixture()
36
-def input_processor_for_qwen():
37
- # Lazy import to avoid initializing CUDA during test collection
38
- from vllm.model_executor.models.qwen import input_processor_for_qwen
39
- return input_processor_for_qwen
40
-
41
-
42
-@pytest.fixture()
43
-def qwen_vl_context() -> InputContext:
44
- """Get an InputContext for Qwen-VL."""
45
- return build_model_context(model_name="Qwen/Qwen-VL",
46
- trust_remote_code=True)
47
-
48
-
49
-# Happy path tests for single/multi-image scenarios for the multimodal
50
-# input processor and mapper, respectively
51
-@pytest.mark.parametrize("num_images", [1, 2])
52
-def test_input_processor_valid_mm_data(input_processor_for_qwen,
53
- qwen_vl_context: InputContext,
54
- num_images: int):
55
- """Happy cases for image inputs to Qwen's multimodal input processor."""
56
- prompt = "".join(
57
- [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
58
- inputs = token_inputs(
59
- prompt=prompt,
60
- # When processing multimodal data for a multimodal model, the qwen
61
- # input processor will overwrite the provided prompt_token_ids with
62
- # the image prompts
63
- prompt_token_ids=[],
64
- multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
65
- )
66
- proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
67
- assert isinstance(proc_inputs, dict)
68
-
69
- # Each image should have one start / stop and a fixed context of 256
70
- proc_tokens = proc_inputs["prompt_token_ids"]
71
- assert proc_tokens.count(IMG_START_ID) == num_images
72
- assert proc_tokens.count(IMG_END_ID) == num_images
73
- assert proc_tokens.count(IMG_PAD_ID) == num_images * TOKS_PER_IMG
74
-
75
-
76
-@pytest.mark.parametrize(
77
- "img_data,expected_shape",
78
- [
79
- # single / multi-image
80
- (SAMPLE_IMAGE, (1, 3, IMG_SIZE, IMG_SIZE)),
81
- (2 * [SAMPLE_IMAGE], (2, 3, IMG_SIZE, IMG_SIZE)),
82
- # single / multi-image embeddings
83
- (torch.rand(
84
- (TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
85
- (torch.rand(
86
- (1, TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
87
- (torch.rand(
88
- (2, TOKS_PER_IMG, VIS_ENC_DIM)), (2, TOKS_PER_IMG, VIS_ENC_DIM)),
89
- ])
90
-def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
91
- qwen_vl_context: InputContext,
92
- img_data: Union[torch.Tensor, List[Image],
93
- Image],
94
- expected_shape: List[int]):
95
- """Happy cases for image inputs to Qwen's multimodal input mapper."""
96
- mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
97
- # Ensure that we get the appropriately shaped pixel_values
98
- # for images and image embeddings, respectively.
99
- assert isinstance(mapped_img_data, MultiModalKwargs)
100
- assert "pixel_values" in mapped_img_data
101
- assert mapped_img_data["pixel_values"].shape == expected_shape
102
-
103
-
104
-# Sad path tests for the multimodal input processor and mapper, respectively
105
-@pytest.mark.parametrize("mm_data", [
106
- {
107
- "image": torch.rand(5)
108
- },
109
- {
110
- "image": torch.rand((5, 5, 5, 5, 5))
111
- },
112
-])
113
-def test_input_processor_invalid_mm_data(input_processor_for_qwen,
114
- qwen_vl_context: InputContext,
115
- mm_data: Dict[str, torch.Tensor]):
116
- """Test sad cases validated in Qwen's multimodal input processor."""
117
- tokenizer = cached_get_tokenizer(qwen_vl_context.model_config.tokenizer,
118
- trust_remote_code=True)
119
- prompt = "Picture 1: <img></img>\n"
120
- prompt_token_ids = tokenizer.encode(prompt)
121
- inputs = token_inputs(prompt=prompt,
122
- prompt_token_ids=prompt_token_ids,
123
- multi_modal_data=mm_data)
124
- # Should fail since we have too many or too few dimensions for embeddings
125
- with pytest.raises(ValueError):
126
- input_processor_for_qwen(qwen_vl_context, inputs)
127
-
128
-
129
-@pytest.mark.parametrize(
130
- "img_data",
131
- [
132
- # Wrong context length
133
- torch.rand((1, TOKS_PER_IMG + 10, VIS_ENC_DIM)),
134
- # Wrong visual encoder output size
135
- torch.rand((1, TOKS_PER_IMG, VIS_ENC_DIM + 10)),
136
- ])
137
-def test_input_mapper_invalid_mm_data(
138
- input_mapper_for_qwen,
139
- qwen_vl_context: InputContext,
140
- img_data: Union[torch.Tensor, List[Image], Image],
141
-):
142
- """Sad cases validated in Qwen VL's multimodal input mapper."""
143
- with pytest.raises(ValueError):
144
- input_mapper_for_qwen(qwen_vl_context, img_data)
vllm/model_executor/models/qwen.py
+354
-300
@@ -4,26 +4,28 @@
4
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
5
"""Inference-only QWen model compatible with HuggingFace weights."""
6
7
+import copy
8
import math
9
import re
9
-from functools import partial
10
-from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
11
- Optional, Set, Tuple, TypedDict, Union)
10
+import unicodedata
11
+from functools import lru_cache, partial
12
+from typing import (AbstractSet, Any, Callable, Collection, Dict, Iterable,
13
+ List, Literal, Mapping, Optional, Set, Tuple, TypedDict,
14
+ Union)
15
13
-import numpy as np
16
import torch
15
-from PIL import Image
17
from torch import nn
18
from torchvision import transforms
19
from torchvision.transforms import InterpolationMode
19
-from transformers import PretrainedConfig
20
+from transformers import (BatchFeature, PretrainedConfig, PreTrainedTokenizer,
21
+ TensorType)
22
+from transformers.image_utils import ImageInput
23
+from transformers.tokenization_utils_base import TextInput
24
25
from vllm.attention import Attention, AttentionMetadata
26
from vllm.compilation.decorators import support_torch_compile
27
from vllm.config import CacheConfig, VllmConfig
28
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
25
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
26
- InputContext, token_inputs)
29
from vllm.logger import init_logger
30
from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
31
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -42,15 +44,20 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
44
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
45
from vllm.model_executor.models.module_mapping import MultiModelKeys
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
46
-from vllm.multimodal.utils import cached_get_tokenizer
47
-from vllm.sequence import IntermediateTensors, SequenceData
48
-from vllm.utils import is_list_of
47
+from vllm.multimodal import MULTIMODAL_REGISTRY
48
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
49
+ NestedTensors)
50
+from vllm.multimodal.parse import MultiModalDataItems
51
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
52
+ BaseProcessingInfo, PromptReplacement,
53
+ PromptReplacementDetails)
54
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
55
+from vllm.sequence import IntermediateTensors
56
57
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
58
from .utils import (flatten_bn, is_pp_missing_parameter,
59
make_empty_intermediate_tensors_factory, make_layers,
53
- maybe_prefix)
60
+ maybe_prefix, merge_multimodal_embeddings)
61
62
logger = init_logger(__name__)
63
@@ -353,8 +360,10 @@ class VisionTransformer(nn.Module):
360
self.ln_post = norm_layer(output_dim)
361
self.proj = nn.Parameter(
362
(output_dim**-0.5) * torch.randn(output_dim, output_dim))
363
+
364
self.image_start_id = image_start_id
365
self.image_end_id = image_start_id + 1
366
+ self.image_pad_id = image_start_id + 2
367
368
def forward(self, x: torch.Tensor) -> torch.Tensor:
369
x = x.to(
@@ -383,21 +392,6 @@ class VisionTransformer(nn.Module):
392
393
return x
394
386
- def get_image_positions(self,
387
- input_ids: torch.Tensor) -> Optional[torch.Tensor]:
388
- """Given the input IDs, extracts start/stop points corresponding to
389
- images.
390
-
391
- args:
392
- Returns:
393
- Optional torch tensor corresponding to start/stop pairs of images.
394
- """
395
- if torch.any(input_ids == self.image_start_id):
396
- bos_pos = torch.where(input_ids == self.image_start_id)
397
- eos_pos = torch.where(input_ids == self.image_end_id)
398
- return torch.stack((bos_pos[0], eos_pos[0]), dim=1)
399
- return None
400
-
395
396
class QWenMLP(nn.Module):
397
"""MLP for the language component of the Qwen model, which contains a
@@ -579,9 +573,12 @@ class QWenModel(nn.Module):
573
self.make_empty_intermediate_tensors = (
574
make_empty_intermediate_tensors_factory(
575
["hidden_states", "residual"], config.hidden_size))
582
- self.visual = VisionTransformer(**config.visual,
583
- quant_config=quant_config) if hasattr(
584
- config, "visual") else None
576
+
577
+ if (vision_config := getattr(config, "visual", None)):
578
+ self.visual = VisionTransformer(**vision_config,
579
+ quant_config=quant_config)
580
+ else:
581
+ self.visual = None
582
583
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
584
return self.wte(input_ids)
@@ -593,38 +590,13 @@ class QWenModel(nn.Module):
590
kv_caches: List[torch.Tensor],
591
attn_metadata: AttentionMetadata,
592
intermediate_tensors: Optional[IntermediateTensors],
596
- pixel_values: Optional[QwenImageInputs],
593
inputs_embeds: Optional[torch.Tensor] = None,
594
) -> Union[torch.Tensor, IntermediateTensors]:
599
- img_pos = None
600
- # If pixel / visual embeddings are provided, this is a visual model
601
- if pixel_values is not None and self.visual is not None:
602
- if pixel_values["type"] != "image_embeds":
603
- image_embeds = self.visual(pixel_values["data"])
604
- else:
605
- image_embeds = pixel_values["data"]
606
-
607
- # features should be of shape (# images, 256, hidden_dim)
608
- img_pos = self.visual.get_image_positions(input_ids)
609
- if isinstance(
610
- img_pos,
611
- np.ndarray) and img_pos.shape[0] != image_embeds.shape[0]:
612
- raise ValueError(
613
- f"Number of placeholders: {img_pos.shape[0]} "
614
- f"does not match number of images {image_embeds.shape[0]}."
615
- )
616
-
595
if get_pp_group().is_first_rank:
596
if inputs_embeds is not None:
597
hidden_states = inputs_embeds
598
else:
599
hidden_states = self.get_input_embeddings(input_ids)
622
- hidden_states = self.wte(input_ids)
623
- # Merge the image embeddings into the hidden states if actually have
624
- # visual features and the corresponding image tokens
625
- if img_pos is not None:
626
- for idx, (img_bos, img_eos) in enumerate(img_pos):
627
- hidden_states[img_bos + 1:img_eos] = image_embeds[idx]
600
residual = None
601
else:
602
assert intermediate_tensors is not None
@@ -648,159 +620,9 @@ class QWenModel(nn.Module):
620
return hidden_states
621
622
651
-def get_image_text(image_num: int, padding: bool) -> str:
652
- """Retrieves a placeholder text that when tokenized, will be expanded with
653
- image pads.
654
-
655
- Args:
656
- image_num: The number of the image that we want a text prompt for.
657
- Images should be indexed starting at 1.
658
- padding: Whether or not padding should be manually added.
659
-
660
- Returns:
661
- Text placeholder prompt for the image being considered.
662
- """
663
- image_start = f"Picture {image_num}: {IMG_START}"
664
- image_end = f"{IMG_END}\n"
665
- if not padding:
666
- return f"{image_start}{image_end}"
667
- return f"{image_start}{MAX_QWEN_IMG_TOKENS * IMG_PAD}{image_end}"
668
-
669
-
670
-def input_processor_for_qwen(ctx: InputContext,
671
- inputs: DecoderOnlyInputs) -> DecoderOnlyInputs:
672
- """Processes the inputs, which may or may not be multimodal.
673
- Multimodal inputs will only be processed if the model has a "visual"
674
- component in its model config, otherwise they'll be ignored.
675
-
676
- Args:
677
- ctx: Context of the loaded model.
678
- inputs: LLM inputs which may have a multi_modal_data attribute.
679
-
680
- Returns:
681
- If the model is language only or not multimodal inputs were provided,
682
- returns inputs unmodified. Otherwise, processes the multimodal
683
- images / image embeddings and adds the fixed-length image placeholders.
684
- """
685
- multi_modal_data = inputs.get("multi_modal_data")
686
-
687
- # Only process images if we have multimodal data and a visual config
688
- hf_config = ctx.get_hf_config()
689
- if (multi_modal_data is None or "image" not in multi_modal_data
690
- or not hasattr(hf_config, "visual")):
691
- return inputs
692
-
693
- prompt = inputs.get("prompt")
694
- prompt_token_ids = inputs["prompt_token_ids"]
695
- model_config = ctx.model_config
696
- tokenizer = cached_get_tokenizer(
697
- model_config.tokenizer,
698
- trust_remote_code=model_config.trust_remote_code)
699
- image_data = multi_modal_data["image"]
700
- if isinstance(image_data, torch.Tensor):
701
- num_dims = len(image_data.shape)
702
- if num_dims < 2 or num_dims > 3:
703
- raise ValueError(
704
- f"Expected img embeds to be have 3 dimensions, got {num_dims}")
705
- num_images = 1 if num_dims == 2 else image_data.shape[0]
706
- elif isinstance(image_data, Image.Image):
707
- num_images = 1
708
- elif is_list_of(image_data, Image.Image):
709
- num_images = len(image_data)
710
- else:
711
- raise TypeError(f"Invalid image type: {type(image_data)}")
712
-
713
- if prompt is None:
714
- prompt = tokenizer.decode(prompt_token_ids)
715
-
716
- # Drops anything between <img>/</img> tags; encoding with the tokenizer
717
- # will automatically add the image pads for the context.
718
- new_prompt, num_matched_images = re.subn(
719
- r"(Picture \d*: <img>).*?(<\/img>\n)",
720
- r"\1\2",
721
- prompt,
722
- )
723
-
724
- if num_matched_images != num_images:
725
- logger.warning(
726
- "Number of matched image placeholders %s doesn't match the number "
727
- "of expected images %s; check your placeholder formatting.",
728
- num_matched_images, num_images)
729
-
730
- new_prompt_token_ids = tokenizer.encode(new_prompt)
731
-
732
- return token_inputs(prompt=new_prompt,
733
- prompt_token_ids=new_prompt_token_ids,
734
- multi_modal_data=multi_modal_data)
735
-
736
-
737
-def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalKwargs:
738
- """Maps the input data to its MultiModalKwargs (if any).
739
-
740
- Args:
741
- ctx: Context of the loaded model.
742
- data: data potentially containing image/image embeddings to be mapped
743
- to pixel_values in .forward() for a visual QWenLMHeadModel model.
744
-
745
- Returns:
746
- MultiModalKwargs containing the stacked normalized images tensor or
747
- image embeddings.
748
- """
749
- # Early exit if we have provided an image to a language only Qwen model
750
- hf_config = ctx.get_hf_config()
751
- if not hasattr(hf_config, "visual"):
752
- logger.warning(
753
- "Images were provided but this model has no visual config; "
754
- "multimodal inputs will not be forwarded to the model.")
755
- return MultiModalKwargs()
756
-
757
- model_config = ctx.model_config
758
- tokenizer = cached_get_tokenizer(
759
- model_config.tokenizer,
760
- trust_remote_code=model_config.trust_remote_code)
761
-
762
- image_pair_tok = tokenizer.encode(IMG_START + IMG_END,
763
- add_special_tokens=False,
764
- return_tensors="pt").squeeze()
765
- image_start_id = image_pair_tok[0]
766
- image_end_id = image_pair_tok[-1]
767
- if (image_start_id + 1) != image_end_id:
768
- raise ValueError(
769
- f"Found image end ID {image_end_id}, but expected {IMG_START} + 1")
770
- if len(image_pair_tok) != (MAX_QWEN_IMG_TOKENS + 2):
771
- raise ValueError(
772
- f"Expected image context length of {MAX_QWEN_IMG_TOKENS}, "
773
- f"but got {image_pair_tok - 2}")
774
-
775
- hf_config = ctx.get_hf_config()
776
- image_size = hf_config.visual["image_size"]
777
- img_emb_size = hf_config.visual["output_dim"]
778
-
779
- if isinstance(data, torch.Tensor):
780
- # It's expected that our values have already been processed
781
- # by the visual transformer; shape is expected to be:
782
- # (# images, 256, hidden_size)
783
- if len(data.shape) == 2:
784
- # Assume only one image embed was provided; unsqueeze the extra dim
785
- data = data.unsqueeze(0)
786
- if len(data.shape) != 3 or data.shape[
787
- 1] != MAX_QWEN_IMG_TOKENS or data.shape[2] != img_emb_size:
788
- raise ValueError(
789
- "Expected image embeds to be a tensor of shape"
790
- f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but "
791
- f"received shape [{data.shape}]")
792
- pixel_values = data
793
- else:
794
- transform = build_normalization_transform(image_size)
795
- if not isinstance(data, (list, tuple)):
796
- data = [data]
797
- transformed_images = [transform(datum) for datum in data]
798
- pixel_values = torch.stack(transformed_images, dim=0)
799
- return MultiModalKwargs({"pixel_values": pixel_values})
800
-
801
-
623
def build_normalization_transform(image_size: int) -> transforms.Compose:
803
- """Builds a normalization transform which can be applied to one or
624
+ """
625
+ Build a normalization transform which can be applied to one or
626
more input images from which we want to extract visual features.
627
628
Args:
@@ -817,62 +639,251 @@ def build_normalization_transform(image_size: int) -> transforms.Compose:
639
])
640
641
820
-def dummy_data_for_qwen(
821
- ctx: InputContext,
822
- seq_len: int,
823
- mm_counts: Mapping[str, int],
824
-) -> DummyData:
825
- """Build dummy data for warming up Qwen models; this will only contain text
826
- matching the defaults for VLLM unless the model has a visual config.
642
+@lru_cache(maxsize=1)
643
+def _get_tokenizer_without_image_pad(
644
+ tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
645
+ """
646
+ The logic of adding image pad tokens should only be applied in
647
+ :class:`QWenVLProcessor`, so they are patched out here.
648
828
- Args:
829
- ctx: Context of the loaded model.
830
- seq_len: Number of tokens in the text sequence.
831
- mm_counts: multimodal data counts.
832
-
833
- Returns:
834
- Tuple containing sequential and multimodal data.
649
+ The definition of the wrapped tokenizer can be found here:
650
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
651
+ """
652
+ new_tokenizer = copy.deepcopy(tokenizer)
653
+
654
+ class TokenizerWithoutImagePad(tokenizer.__class__): # type: ignore
655
+
656
+ def tokenize(
657
+ self,
658
+ text: str,
659
+ allowed_special: Union[AbstractSet[str], str] = "all",
660
+ disallowed_special: Union[Collection[str], str] = (),
661
+ **kwargs,
662
+ ) -> list[Union[bytes, str]]:
663
+ text = unicodedata.normalize("NFC", text)
664
+
665
+ return [
666
+ self.decoder[t] for t in self.tokenizer.encode(
667
+ text,
668
+ allowed_special=allowed_special,
669
+ disallowed_special=disallowed_special,
670
+ )
671
+ ]
672
+
673
+ def _decode(
674
+ self,
675
+ token_ids: Union[int, List[int]],
676
+ skip_special_tokens: bool = False,
677
+ errors: Optional[str] = None,
678
+ **kwargs,
679
+ ) -> str:
680
+ if isinstance(token_ids, int):
681
+ token_ids = [token_ids]
682
+
683
+ return self.tokenizer.decode(
684
+ token_ids,
685
+ errors=errors or self.errors,
686
+ )
687
+
688
+ TokenizerWithoutImagePad.__name__ = \
689
+ f"{tokenizer.__class__.__name__}WithoutImagePad"
690
+
691
+ new_tokenizer.__class__ = TokenizerWithoutImagePad
692
+ return new_tokenizer
693
+
694
+
695
+class QWenVLProcessor:
696
+ """
697
+ This model doesn't define its own HF processor,
698
+ so we implement our own one here.
699
+
700
+ We call the wrapped tokenizer to automatically insert image pad tokens:
701
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py#L245
702
+
703
+ The image processor is defined here:
704
+ https://huggingface.co/Qwen/Qwen-VL/blob/main/visual.py#L354
705
"""
836
- hf_config = ctx.get_hf_config()
837
-
838
- # The presence of a visual config indicates this is a multimodal model.
839
- # If we don't have it, the model is considered an LLM for warmup purposes.
840
- if not hasattr(hf_config, "visual"):
841
- seq_data = SequenceData.from_prompt_token_counts((0, seq_len))
842
- mm_data = None
843
- return DummyData(seq_data, mm_data)
844
-
845
- # We have a visual component - use images to warm up
846
- num_images = mm_counts["image"]
847
- model_config = ctx.model_config
848
- tokenizer = cached_get_tokenizer(
849
- model_config.tokenizer,
850
- trust_remote_code=model_config.trust_remote_code)
851
-
852
- # Build the image prompts with no imgpads; the tokenizer will add img pads
853
- image_prompt = ''.join(
854
- [get_image_text(idx, False) for idx in range(1, num_images + 1)])
855
- toks = tokenizer.encode(image_prompt, add_special_tokens=False)
856
-
857
- # Make sure we actually get the fixed context size per tok padding
858
- num_pads = toks.count(tokenizer.encode(IMG_PAD)[0])
859
- if num_pads != (num_images * MAX_QWEN_IMG_TOKENS):
860
- raise ValueError(
861
- f"Tokenized dummy data should encode {MAX_QWEN_IMG_TOKENS} pads"
862
- f" per image, but got {num_pads} pads for {num_images} image(s)"
863
- " in total. Are you using a qwen tokenizer?")
864
-
865
- # Ensure the number of tokens is at minimum the sequence length provided
866
- if len(toks) < seq_len:
867
- toks += [0] * (seq_len - len(toks))
868
-
869
- seq_data = SequenceData.from_seqs(toks)
870
-
871
- # Build the input images; width/height doesn't actually matter here since
872
- # the data will get resized and the # of tokens per image is constant
873
- image = Image.new("RGB", (224, 224), color=0)
874
- mm_data = {"image": image if num_images == 1 else [image] * num_images}
875
- return DummyData(seq_data, mm_data)
706
+
707
+ def __init__(
708
+ self,
709
+ config: PretrainedConfig,
710
+ tokenizer: PreTrainedTokenizer,
711
+ ) -> None:
712
+ super().__init__()
713
+
714
+ self.config = config
715
+ self.tokenizer = tokenizer
716
+
717
+ if hasattr(self.config, "visual"):
718
+ self.image_transform = build_normalization_transform(
719
+ config.visual["image_size"])
720
+ else:
721
+ self.image_transform = None
722
+
723
+ special_tokens: dict[str,
724
+ int] = tokenizer.special_tokens # type: ignore
725
+ self.img_start_id = special_tokens[IMG_START]
726
+ self.img_end_id = special_tokens[IMG_END]
727
+
728
+ def __call__(
729
+ self,
730
+ text: Optional[Union[TextInput, list[TextInput]]] = None,
731
+ images: Optional[Union[ImageInput, list[ImageInput]]] = None,
732
+ return_tensors: Optional[Union[str, TensorType]] = None,
733
+ ) -> BatchFeature:
734
+ if text is None:
735
+ text = []
736
+ if not isinstance(text, list):
737
+ text = [text]
738
+ if images is None:
739
+ images = []
740
+ if not isinstance(images, list):
741
+ images = [images]
742
+
743
+ text_inputs = self.tokenizer(text)
744
+
745
+ if len(images) == 0:
746
+ image_inputs = {}
747
+ else:
748
+ if self.image_transform is None:
749
+ raise ValueError("This model does not support image inputs")
750
+
751
+ pixel_values = [self.image_transform(image) for image in images]
752
+ image_inputs = {"pixel_values": torch.stack(pixel_values)}
753
+
754
+ return BatchFeature(
755
+ {
756
+ **text_inputs,
757
+ **image_inputs,
758
+ },
759
+ tensor_type=return_tensors,
760
+ )
761
+
762
+
763
+class QWenVLProcessingInfo(BaseProcessingInfo):
764
+
765
+ def get_tokenizer(self) -> PreTrainedTokenizer:
766
+ tokenizer = self.ctx.tokenizer
767
+ assert isinstance(tokenizer, PreTrainedTokenizer)
768
+
769
+ return _get_tokenizer_without_image_pad(tokenizer)
770
+
771
+ def get_hf_processor(self) -> QWenVLProcessor:
772
+ tokenizer = self.ctx.tokenizer
773
+ assert isinstance(tokenizer, PreTrainedTokenizer)
774
+
775
+ return QWenVLProcessor(self.get_hf_config(), tokenizer)
776
+
777
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
778
+ return {"image": None}
779
+
780
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
781
+ return {"image": self.get_num_image_tokens()}
782
+
783
+ def get_num_image_tokens(self) -> int:
784
+ return MAX_QWEN_IMG_TOKENS
785
+
786
+
787
+class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
788
+
789
+ def get_dummy_processor_inputs(
790
+ self,
791
+ seq_len: int,
792
+ mm_counts: Mapping[str, int],
793
+ ) -> ProcessorInputs:
794
+ hf_config = self.info.get_hf_config()
795
+ if not hasattr(hf_config, "visual"):
796
+ return ProcessorInputs(prompt_text="", mm_data={})
797
+
798
+ vision_config = hf_config.visual
799
+
800
+ max_image_size = vision_config["image_size"]
801
+ num_images = mm_counts.get("image", 0)
802
+
803
+ mm_data = {
804
+ "image":
805
+ self._get_dummy_images(width=max_image_size,
806
+ height=max_image_size,
807
+ num_images=num_images)
808
+ }
809
+
810
+ return ProcessorInputs(
811
+ prompt_text="".join(f"Picture {i}: {IMG_START}{IMG_END}\n"
812
+ for i in range(1, num_images + 1)),
813
+ mm_data=mm_data,
814
+ )
815
+
816
+
817
+class QWenVLMultiModalProcessor(BaseMultiModalProcessor[QWenVLProcessingInfo]):
818
+
819
+ def _call_hf_processor(
820
+ self,
821
+ prompt: str,
822
+ mm_data: Mapping[str, object],
823
+ mm_kwargs: Mapping[str, object],
824
+ ) -> BatchFeature:
825
+ # Drops anything between <img>/</img> tags; encoding with the tokenizer
826
+ # will automatically add the image pads for the context.
827
+ prompt, num_matched_images = re.subn(
828
+ r"(Picture \d*: <img>).*?(<\/img>\n)",
829
+ r"\1\2",
830
+ prompt,
831
+ )
832
+
833
+ image_data = mm_data.get("images")
834
+ if image_data is not None:
835
+ assert isinstance(image_data, list)
836
+
837
+ num_images = len(image_data)
838
+ if num_matched_images != num_images:
839
+ logger.warning(
840
+ "Number of matched image placeholders %s doesn't match "
841
+ "the number of expected images %s; check your placeholder "
842
+ "formatting.", num_matched_images, num_images)
843
+
844
+ return super()._call_hf_processor(
845
+ prompt=prompt,
846
+ mm_data=mm_data,
847
+ mm_kwargs=mm_kwargs,
848
+ )
849
+
850
+ def _get_mm_fields_config(
851
+ self,
852
+ hf_inputs: BatchFeature,
853
+ hf_processor_mm_kwargs: Mapping[str, object],
854
+ ) -> Mapping[str, MultiModalFieldConfig]:
855
+ return dict(
856
+ pixel_values=MultiModalFieldConfig.batched("image"),
857
+ image_embeds=MultiModalFieldConfig.batched("image"),
858
+ )
859
+
860
+ def _get_prompt_replacements(
861
+ self,
862
+ mm_items: MultiModalDataItems,
863
+ hf_processor_mm_kwargs: Mapping[str, object],
864
+ out_mm_kwargs: MultiModalKwargs,
865
+ ) -> list[PromptReplacement]:
866
+ tokenizer = self.info.get_tokenizer()
867
+ special_tokens: dict[str,
868
+ int] = tokenizer.special_tokens # type: ignore
869
+
870
+ img_start_id = special_tokens[IMG_START]
871
+ img_end_id = special_tokens[IMG_END]
872
+ img_pad_id = special_tokens[IMG_PAD]
873
+
874
+ num_image_tokens = self.info.get_num_image_tokens()
875
+ image_tokens = [img_pad_id] * num_image_tokens
876
+
877
+ return [
878
+ PromptReplacement(
879
+ modality="image",
880
+ target=[img_start_id, img_end_id],
881
+ replacement=PromptReplacementDetails(
882
+ full=[img_start_id] + image_tokens + [img_end_id],
883
+ features=image_tokens,
884
+ ),
885
+ )
886
+ ]
887
888
889
class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
@@ -898,38 +909,77 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
909
self.make_empty_intermediate_tensors = (
910
self.transformer.make_empty_intermediate_tensors)
911
901
- def _get_image_input_type(
902
- self,
903
- pixel_values: Optional[torch.Tensor]) -> Optional[QwenImageInputs]:
904
- """Determines if the provided pixel_values are normalized pixel values
905
- or image embeddings.
912
+ def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
913
+ h = w = self.config.visual["image_size"]
914
+ expected_dims = (3, h, w)
915
+ actual_dims = tuple(data.shape[1:])
916
907
- Args:
908
- pixel_values: Optional data to processed into visual embeddings.
917
+ if actual_dims != expected_dims:
918
+ expected_expr = ("batch_size", *map(str, expected_dims))
919
+ raise ValueError(
920
+ f"The expected shape of pixel values is {expected_expr}. "
921
+ f"You supplied {tuple(data.shape)}.")
922
+
923
+ return data
924
+
925
+ def _parse_and_validate_image_input(
926
+ self, **kwargs: object) -> Optional[QwenImageInputs]:
927
+ pixel_values = kwargs.pop("pixel_values", None)
928
+ image_embeds = kwargs.pop("image_embeds", None)
929
+
930
+ if pixel_values is not None:
931
+ if not isinstance(pixel_values, torch.Tensor):
932
+ raise ValueError("Incorrect type of pixel values. "
933
+ f"Got type: {type(pixel_values)}")
934
+
935
+ return QwenImagePixelInputs(
936
+ type="pixel_values",
937
+ data=self._validate_pixel_values(
938
+ flatten_bn(pixel_values, concat=True)),
939
+ )
940
+
941
+ if image_embeds is not None:
942
+ if not isinstance(image_embeds, torch.Tensor):
943
+ raise ValueError("Incorrect type of image embeddings. "
944
+ f"Got type: {type(image_embeds)}")
945
+
946
+ return QwenImageEmbeddingInputs(
947
+ type="image_embeds",
948
+ data=flatten_bn(image_embeds),
949
+ )
950
910
- Returns:
911
- None of the QwenImageInputs type used to determine whether or not
912
- the visual transformer needs to process the pixel_values.
913
- """
914
- if pixel_values is not None and self.transformer.visual is not None:
915
- pixel_values = flatten_bn(pixel_values)
916
- if len(pixel_values.shape) == 3 and pixel_values.shape[
917
- 1] == MAX_QWEN_IMG_TOKENS and pixel_values.shape[
918
- 2] == self.config.visual["output_dim"]:
919
- return QwenImageEmbeddingInputs(
920
- type="image_embeds",
921
- data=pixel_values,
922
- )
923
- else:
924
- # If we have the wrong shape, assume we still need to process
925
- return QwenImagePixelInputs(
926
- type="pixel_values",
927
- data=pixel_values,
928
- )
951
return None
952
931
- def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
932
- return self.transformer.get_input_embeddings(input_ids)
953
+ def _process_image_input(self,
954
+ image_input: QwenImageInputs) -> torch.Tensor:
955
+ if image_input["type"] == "image_embeds":
956
+ return image_input["data"]
957
+
958
+ assert self.transformer.visual is not None
959
+ return self.transformer.visual(image_input["data"])
960
+
961
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
962
+ image_input = self._parse_and_validate_image_input(**kwargs)
963
+ if image_input is None:
964
+ return None
965
+
966
+ vision_embeddings = self._process_image_input(image_input)
967
+ return vision_embeddings
968
+
969
+ def get_input_embeddings(
970
+ self,
971
+ input_ids: torch.Tensor,
972
+ multimodal_embeddings: Optional[NestedTensors] = None,
973
+ ) -> torch.Tensor:
974
+ inputs_embeds = self.transformer.get_input_embeddings(input_ids)
975
+
976
+ if multimodal_embeddings is not None:
977
+ assert self.transformer.visual is not None
978
+ inputs_embeds = merge_multimodal_embeddings(
979
+ input_ids, inputs_embeds, multimodal_embeddings,
980
+ self.transformer.visual.image_pad_id)
981
+
982
+ return inputs_embeds
983
984
def forward(
985
self,
@@ -938,18 +988,23 @@ class QWenBaseModel(nn.Module, SupportsPP, SupportsLoRA):
988
kv_caches: List[torch.Tensor],
989
attn_metadata: AttentionMetadata,
990
intermediate_tensors: Optional[IntermediateTensors] = None,
941
- pixel_values: Optional[torch.Tensor] = None,
991
inputs_embeds: Optional[torch.Tensor] = None,
992
+ **kwargs: object,
993
) -> Union[torch.Tensor, IntermediateTensors]:
994
if intermediate_tensors is not None:
995
+ inputs_embeds = None
996
+
997
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
998
+ # condition is for v0 compatibility.
999
+ elif inputs_embeds is None:
1000
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
1001
+ inputs_embeds = self.get_input_embeddings(input_ids,
1002
+ vision_embeddings)
1003
input_ids = None
946
- pixel_values = None
947
- else:
948
- pixel_values = self._get_image_input_type(pixel_values)
1004
1005
hidden_states = self.transformer(input_ids, positions, kv_caches,
1006
attn_metadata, intermediate_tensors,
952
- pixel_values, inputs_embeds)
1007
+ inputs_embeds)
1008
return hidden_states
1009
1010
def compute_logits(
@@ -1063,10 +1118,9 @@ class QWenVL(QWenBaseModel, SupportsMultiModal):
1118
tower_model="transformer.visual.transformer")
1119
1120
1066
-@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
1067
-@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS)
1068
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen)
1069
-@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen)
1121
+@MULTIMODAL_REGISTRY.register_processor(QWenVLMultiModalProcessor,
1122
+ info=QWenVLProcessingInfo,
1123
+ dummy_inputs=QWenVLDummyInputsBuilder)
1124
class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1125
"""
1126
QWenLMHeadModel is not only applicable to LLM but also to VL, which is not
@@ -1084,7 +1138,7 @@ class QWenLMHeadModel(QWenBaseModel, SupportsMultiModal, SupportsLoRA):
1138
cls,
1139
vllm_config: VllmConfig,
1140
prefix: str = "",
1087
- ) -> None:
1141
+ ) -> QWenBaseModel:
1142
config = vllm_config.model_config.hf_config
1143
# Initialize VL
1144
if hasattr(config, "visual"):