A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Clean up and fix multi-modal processors (#13012)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Feb 10, 2025 at 18:45 UTC
51f0b5f7f6ec4aa8199f12bb7df08c9cb5e025db
7 files changed
+124
-154
docs/source/features/compatibility_matrix.md
+1
-1
@@ -297,7 +297,7 @@ Check the '✗' with links to see tracking issue for unsupported feature/hardwar
297
* ✅
298
* ✅
299
* ?
300
- * [✗](gh-issue:7968>)
300
+ * [✗](gh-issue:7968)
301
* ?
302
* ✅
303
*
tests/models/decoder_only/language/test_models.py
+10
@@ -26,6 +26,9 @@ from ...utils import check_logprobs_close
26
"google/gemma-1.1-2b-it", # gemma
27
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
28
),
29
+ pytest.param(
30
+ "THUDM/chatglm3-6b", # ChatGLM (text-only)
31
+ ),
32
pytest.param(
33
"meta-llama/Llama-3.2-1B-Instruct", # llama
34
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
@@ -43,6 +46,9 @@ from ...utils import check_logprobs_close
46
"microsoft/phi-2", # phi
47
marks=[pytest.mark.core_model],
48
),
49
+ pytest.param(
50
+ "Qwen/Qwen-7B", # qwen (text-only)
51
+ ),
52
pytest.param(
53
"Qwen/Qwen2.5-0.5B-Instruct", # qwen2
54
marks=[pytest.mark.core_model],
@@ -68,6 +74,10 @@ def test_models(
74
) -> None:
75
76
with hf_runner(model, dtype=dtype) as hf_model:
77
+ if model.startswith("THUDM/chatglm3"):
78
+ hf_model.model.get_output_embeddings = lambda: \
79
+ hf_model.model.transformer.output_layer
80
+
81
hf_outputs = hf_model.generate_greedy_logprobs_limit(
82
example_prompts, max_tokens, num_logprobs)
83
tests/models/multimodal/processing/test_common.py
+1
-1
@@ -89,7 +89,7 @@ def _test_processing_correctness(
89
mm_data = {
90
k:
91
[(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
92
- for _ in range(rng.randint(limit))]
92
+ for _ in range(rng.randint(limit + 1))]
93
for k, limit in limit_mm_per_prompt.items()
94
}
95
tests/multimodal/utils.py
-3
@@ -17,10 +17,7 @@ def random_video(
17
min_wh: int,
18
max_wh: int,
19
):
20
- # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
20
num_frames = rng.randint(min_frames, max_frames)
22
- num_frames = (num_frames // 2) * 2
23
-
21
w, h = rng.randint(min_wh, max_wh, size=(2, ))
22
return rng.randint(0, 255, size=(num_frames, w, h, 3), dtype=np.uint8)
23
vllm/model_executor/models/chatglm.py
+63
-97
@@ -4,8 +4,8 @@
4
# https://github.com/THUDM/CogAgent
5
"""Inference-only CogAgent model compatible with THUDM weights."""
6
from argparse import Namespace
7
-from typing import (Iterable, List, Mapping, Optional, Sequence, Set, Tuple,
8
- TypedDict, Union)
7
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
8
+ Union)
9
10
import torch
11
from torch import nn
@@ -19,7 +19,6 @@ from transformers.tokenization_utils_base import TextInput
19
from vllm.attention import Attention, AttentionMetadata
20
from vllm.config import CacheConfig, VllmConfig
21
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
22
-from vllm.logger import init_logger
22
from vllm.model_executor.layers.activation import SiluAndMul
23
from vllm.model_executor.layers.layernorm import RMSNorm
24
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -37,12 +36,10 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
38
from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
40
-from vllm.multimodal.parse import ImageSize, MultiModalDataItems
39
+from vllm.multimodal.parse import MultiModalDataItems
40
from vllm.multimodal.processing import (BaseMultiModalProcessor,
41
BaseProcessingInfo, BatchFeature,
43
- BoundPromptReplacement,
42
MultiModalFieldConfig,
45
- PlaceholderFeaturesInfo,
43
PromptReplacement)
44
from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
45
from vllm.sequence import IntermediateTensors
@@ -53,39 +50,6 @@ from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
50
make_empty_intermediate_tensors_factory, make_layers,
51
maybe_prefix, merge_multimodal_embeddings)
52
56
-logger = init_logger(__name__)
57
-
58
-IMAGE_TOKEN_ID = 151329
59
-
60
-
61
-def build_normalization_transform(image_size: int) -> transforms.Compose:
62
- """
63
- Build a normalization transform which can be applied to one or
64
- more input images from which we want to extract visual features.
65
-
66
- Args:
67
- image_size: size of the image to be processed for visual embeddings.
68
-
69
- Returns:
70
- Callable transform for normalizing and resizing one RGB image.
71
- """
72
-
73
- return transforms.Compose([
74
- transforms.Resize(
75
- (image_size, image_size),
76
- interpolation=InterpolationMode.BICUBIC,
77
- ),
78
- transforms.ToTensor(),
79
- transforms.Normalize(
80
- (0.48145466, 0.4578275, 0.40821073),
81
- (0.26862954, 0.26130258, 0.27577711),
82
- ),
83
- ])
84
-
85
-
86
-def calculate_image_placeholder(vision_config):
87
- return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
88
-
53
54
class GLMImagePixelInputs(TypedDict):
55
pixel_values: torch.Tensor
@@ -109,9 +73,20 @@ class GLM4VProcessor:
73
self.config = config
74
self.tokenizer = tokenizer
75
112
- if hasattr(self.config, "vision_config"):
113
- self.image_transform = build_normalization_transform(
114
- config.vision_config["image_size"])
76
+ if vision_config := getattr(config, "vision_config", None):
77
+ image_size = vision_config["image_size"]
78
+
79
+ self.image_transform = transforms.Compose([
80
+ transforms.Resize(
81
+ (image_size, image_size),
82
+ interpolation=InterpolationMode.BICUBIC,
83
+ ),
84
+ transforms.ToTensor(),
85
+ transforms.Normalize(
86
+ mean=(0.48145466, 0.4578275, 0.40821073),
87
+ std=(0.26862954, 0.26130258, 0.27577711),
88
+ ),
89
+ ])
90
else:
91
self.image_transform = None
92
@@ -150,9 +125,19 @@ class GLM4VProcessor:
125
126
class GLM4VProcessingInfo(BaseProcessingInfo):
127
153
- def __init__(self, ctx):
154
- super().__init__(ctx)
155
- self._pre_calculate()
128
+ def get_tokenizer(self):
129
+ tokenizer = self.ctx.tokenizer
130
+ assert isinstance(tokenizer, PreTrainedTokenizer)
131
+ return tokenizer
132
+
133
+ def get_hf_config(self):
134
+ return self.ctx.get_hf_config(ChatGLMConfig)
135
+
136
+ def get_hf_processor(self) -> GLM4VProcessor:
137
+ return GLM4VProcessor(
138
+ self.get_hf_config(),
139
+ self.get_tokenizer(),
140
+ )
141
142
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
143
return {"image": 1}
@@ -162,27 +147,21 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
147
seq_len: int,
148
mm_counts: Mapping[str, int],
149
) -> Mapping[str, int]:
150
+ return {"image": self.get_num_image_feature_tokens()}
151
166
- return {"image": self.image_token_num + 2}
167
-
168
- def _pre_calculate(self):
152
+ def get_num_image_tokens(self) -> int:
153
hf_config = self.get_hf_config()
170
- vision_config = hf_config.vision_config
171
- self.image_token_num = calculate_image_placeholder(vision_config)
172
- self.image_size = vision_config["image_size"]
154
+ if not (vision_config := getattr(hf_config, "vision_config", None)):
155
+ return 0
156
174
- def get_num_image_tokens(self) -> int:
175
- return self.image_token_num + 2
157
+ image_size = vision_config["image_size"]
158
+ patch_size = vision_config["patch_size"]
159
+ grid_length = image_size // patch_size // 2
160
+ return grid_length * grid_length
161
177
- def get_image_size(self) -> ImageSize:
178
-
179
- return ImageSize(height=self.image_size, width=self.image_size)
180
-
181
- def get_hf_processor(self) -> GLM4VProcessor:
182
- return GLM4VProcessor(
183
- self.get_hf_config(),
184
- self.get_tokenizer(),
185
- )
162
+ def get_num_image_feature_tokens(self) -> int:
163
+ # EVA2CLIPModel has embeddings for boi and eoi tokens as well
164
+ return self.get_num_image_tokens() + 2
165
166
167
class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
@@ -192,8 +171,12 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
171
seq_len: int,
172
mm_counts: Mapping[str, int],
173
) -> ProcessorInputs:
174
+ hf_config = self.info.get_hf_config()
175
+ if not (vision_config := getattr(hf_config, "vision_config", None)):
176
+ return ProcessorInputs(prompt_text="", mm_data={})
177
+
178
+ target_width = target_height = vision_config["image_size"]
179
num_images = mm_counts.get("image", 0)
196
- target_width, target_height = self.info.get_image_size()
180
181
mm_data = {
182
"image":
@@ -201,9 +184,11 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
184
height=target_height,
185
num_images=num_images)
186
}
204
- text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
187
+
188
+ base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
189
+
190
return ProcessorInputs(
206
- prompt_text=text,
191
+ prompt_text=base_text * num_images,
192
mm_data=mm_data,
193
)
194
@@ -223,47 +208,28 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
208
hf_processor_mm_kwargs: Mapping[str, object],
209
out_mm_kwargs: MultiModalKwargs,
210
) -> list[PromptReplacement]:
211
+ hf_config = self.info.get_hf_config()
212
+ if not hasattr(hf_config, "vision_config"):
213
+ return []
214
+
215
+ boi_token_id = hf_config.boi_token_id
216
+ image_token_id = hf_config.pad_token_id
217
+ eoi_token_id = hf_config.eoi_token_id
218
219
def get_replacement(item_idx: int):
228
- image_tokens = self.info.image_token_num
229
- return [IMAGE_TOKEN_ID] * image_tokens
220
+ num_image_tokens = self.info.get_num_image_tokens()
221
+ image_tokens = [image_token_id] * num_image_tokens
222
+
223
+ return [boi_token_id] + image_tokens + [eoi_token_id]
224
225
return [
226
PromptReplacement(
227
modality="image",
234
- target=[IMAGE_TOKEN_ID],
228
+ target=[boi_token_id, image_token_id, eoi_token_id],
229
replacement=get_replacement,
230
),
231
]
232
239
- def _apply_prompt_replacements(
240
- self,
241
- token_ids: list[int],
242
- mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
243
- mm_item_counts: Mapping[str, int],
244
- ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
245
- token_ids, text, placeholders = super()._apply_prompt_replacements(
246
- token_ids=token_ids,
247
- mm_prompt_repls=mm_prompt_repls,
248
- mm_item_counts=mm_item_counts,
249
- )
250
- hf_config = self.info.get_hf_config()
251
- boi_token_id = hf_config.boi_token_id
252
- eoi_token_id = hf_config.eoi_token_id
253
- placeholders = {
254
- modality: [
255
- PlaceholderFeaturesInfo(
256
- modality=p.modality,
257
- item_idx=p.item_idx,
258
- start_idx=p.start_idx - 1,
259
- tokens=[boi_token_id] + p.tokens + [eoi_token_id],
260
- ) for p in ps
261
- ]
262
- for modality, ps in placeholders.items()
263
- }
264
-
265
- return token_ids, text, placeholders
266
-
233
234
class GLMAttention(nn.Module):
235
@@ -618,7 +584,7 @@ class ChatGLMModel(nn.Module):
584
multimodal_embeddings=multimodal_embeddings,
585
placeholder_token_id=[
586
self.config.boi_token_id,
621
- IMAGE_TOKEN_ID,
587
+ self.config.pad_token_id,
588
self.config.eoi_token_id,
589
],
590
)
vllm/model_executor/models/qwen.py
+46
-45
@@ -63,18 +63,6 @@ from .utils import (flatten_bn, is_pp_missing_parameter,
63
64
logger = init_logger(__name__)
65
66
-# NOTE: Qwen models have a few other special tags, e.g., ref, bbox, quad;
67
-# for the time being, these tags are not considered as special at encoding
68
-# time. This may change as VLLMs multimodal API changes in the future.
69
-IMG_START = "<img>"
70
-IMG_END = "</img>"
71
-IMG_PAD = "<imgpad>"
72
-# Image context is fixed at 256 for all images
73
-MAX_QWEN_IMG_TOKENS = 256
74
-# Image normalization params
75
-CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
76
-CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
77
-
66
67
class QwenImagePixelInputs(TypedDict):
68
type: Literal["pixel_values"]
@@ -622,25 +610,6 @@ class QWenModel(nn.Module):
610
return hidden_states
611
612
625
-def build_normalization_transform(image_size: int) -> transforms.Compose:
626
- """
627
- Build a normalization transform which can be applied to one or
628
- more input images from which we want to extract visual features.
629
-
630
- Args:
631
- image_size: size of the image to be processed for visual embeddings.
632
-
633
- Returns:
634
- Callable transform for normalizing and resizing one RGB image.
635
- """
636
- return transforms.Compose([
637
- transforms.Resize((image_size, image_size),
638
- interpolation=InterpolationMode.BICUBIC),
639
- transforms.ToTensor(),
640
- transforms.Normalize(mean=CLIP_MEAN, std=CLIP_STD),
641
- ])
642
-
643
-
613
@lru_cache(maxsize=1)
614
def _get_tokenizer_without_image_pad(
615
tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
@@ -716,16 +685,34 @@ class QWenVLProcessor:
685
self.config = config
686
self.tokenizer = tokenizer
687
719
- if hasattr(self.config, "visual"):
720
- self.image_transform = build_normalization_transform(
721
- config.visual["image_size"])
688
+ if vision_config := getattr(self.config, "visual", None):
689
+ image_size = vision_config["image_size"]
690
+
691
+ self.image_transform = transforms.Compose([
692
+ transforms.Resize(
693
+ (image_size, image_size),
694
+ interpolation=InterpolationMode.BICUBIC,
695
+ ),
696
+ transforms.ToTensor(),
697
+ transforms.Normalize(
698
+ mean=(0.48145466, 0.4578275, 0.40821073),
699
+ std=(0.26862954, 0.26130258, 0.27577711),
700
+ ),
701
+ ])
702
else:
703
self.image_transform = None
704
725
- special_tokens: dict[str,
726
- int] = tokenizer.special_tokens # type: ignore
727
- self.img_start_id = special_tokens[IMG_START]
728
- self.img_end_id = special_tokens[IMG_END]
705
+ @property
706
+ def image_start_tag(self) -> str:
707
+ return self.tokenizer.image_start_tag # type: ignore
708
+
709
+ @property
710
+ def image_end_tag(self) -> str:
711
+ return self.tokenizer.image_end_tag # type: ignore
712
+
713
+ @property
714
+ def image_pad_tag(self) -> str:
715
+ return self.tokenizer.image_pad_tag # type: ignore
716
717
def __call__(
718
self,
@@ -787,7 +774,14 @@ class QWenVLProcessingInfo(BaseProcessingInfo):
774
return {"image": self.get_num_image_tokens()}
775
776
def get_num_image_tokens(self) -> int:
790
- return MAX_QWEN_IMG_TOKENS
777
+ hf_config = self.get_hf_config()
778
+ if not (vision_config := getattr(hf_config, "visual", None)):
779
+ return 0
780
+
781
+ image_size = vision_config["image_size"]
782
+ patch_size = vision_config["patch_size"]
783
+ grid_length = image_size // patch_size // 2
784
+ return grid_length * grid_length
785
786
787
class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
@@ -798,10 +792,12 @@ class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
792
mm_counts: Mapping[str, int],
793
) -> ProcessorInputs:
794
hf_config = self.info.get_hf_config()
801
- if not hasattr(hf_config, "visual"):
795
+ if not (vision_config := getattr(hf_config, "visual", None)):
796
return ProcessorInputs(prompt_text="", mm_data={})
797
804
- vision_config = hf_config.visual
798
+ processor = self.info.get_hf_processor()
799
+ img_start = processor.image_start_tag
800
+ img_end = processor.image_end_tag
801
802
target_width = target_height = vision_config["image_size"]
803
num_images = mm_counts.get("image", 0)
@@ -814,7 +810,7 @@ class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
810
}
811
812
return ProcessorInputs(
817
- prompt_text="".join(f"Picture {i}: {IMG_START}{IMG_END}\n"
813
+ prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
814
for i in range(1, num_images + 1)),
815
mm_data=mm_data,
816
)
@@ -869,13 +865,18 @@ class QWenVLMultiModalProcessor(BaseMultiModalProcessor[QWenVLProcessingInfo]):
865
hf_processor_mm_kwargs: Mapping[str, object],
866
out_mm_kwargs: MultiModalKwargs,
867
) -> list[PromptReplacement]:
868
+ hf_config = self.info.get_hf_config()
869
+ if not hasattr(hf_config, "visual"):
870
+ return []
871
+
872
tokenizer = self.info.get_tokenizer()
873
special_tokens: dict[str,
874
int] = tokenizer.special_tokens # type: ignore
875
876
- img_start_id = special_tokens[IMG_START]
877
- img_end_id = special_tokens[IMG_END]
878
- img_pad_id = special_tokens[IMG_PAD]
876
+ processor = self.info.get_hf_processor()
877
+ img_start_id = special_tokens[processor.image_start_tag]
878
+ img_end_id = special_tokens[processor.image_end_tag]
879
+ img_pad_id = special_tokens[processor.image_pad_tag]
880
881
num_image_tokens = self.info.get_num_image_tokens()
882
image_tokens = [img_pad_id] * num_image_tokens
vllm/model_executor/models/qwen2_vl.py
+3
-7
@@ -885,14 +885,10 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
885
max_image_tokens = self.get_max_image_tokens() * max_images
886
max_total_frames = self._get_max_video_frames(seq_len -
887
max_image_tokens)
888
- num_frames = min(max(max_total_frames // max(max_videos, 1), 1),
889
- _MAX_FRAMES_PER_VIDEO)
888
+ max_frames_per_video = min(max_total_frames // max(max_videos, 1),
889
+ _MAX_FRAMES_PER_VIDEO)
890
891
- # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
892
- if num_frames > 1 and num_frames % 2 == 1:
893
- num_frames += 1
894
-
895
- return num_frames
891
+ return max(max_frames_per_video, 1)
892
893
def get_max_video_tokens(self, seq_len: int) -> int:
894
target_width, target_height = self.get_image_size_with_most_features()