@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51f0b5f7f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix] Clean up and fix multi-modal processors (#13012)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 10, 2025 at 18:45 UTC 51f0b5f7f6ec4aa8199f12bb7df08c9cb5e025db
7 files changed +124 -154
docs/source/features/compatibility_matrix.md
+1 -1
@@ -297,7 +297,7 @@ Check the '✗' with links to see tracking issue for unsupported feature/hardwar
297 * ✅
298 * ✅
299 * ?
300 - * [✗](gh-issue:7968>)
300 + * [✗](gh-issue:7968)
301 * ?
302 * ✅
303 *
tests/models/decoder_only/language/test_models.py
+10
@@ -26,6 +26,9 @@ from ...utils import check_logprobs_close
26 "google/gemma-1.1-2b-it", # gemma
27 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
28 ),
29 + pytest.param(
30 + "THUDM/chatglm3-6b", # ChatGLM (text-only)
31 + ),
32 pytest.param(
33 "meta-llama/Llama-3.2-1B-Instruct", # llama
34 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
@@ -43,6 +46,9 @@ from ...utils import check_logprobs_close
46 "microsoft/phi-2", # phi
47 marks=[pytest.mark.core_model],
48 ),
49 + pytest.param(
50 + "Qwen/Qwen-7B", # qwen (text-only)
51 + ),
52 pytest.param(
53 "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
54 marks=[pytest.mark.core_model],
@@ -68,6 +74,10 @@ def test_models(
74 ) -> None:
75
76 with hf_runner(model, dtype=dtype) as hf_model:
77 + if model.startswith("THUDM/chatglm3"):
78 + hf_model.model.get_output_embeddings = lambda: \
79 + hf_model.model.transformer.output_layer
80 +
81 hf_outputs = hf_model.generate_greedy_logprobs_limit(
82 example_prompts, max_tokens, num_logprobs)
83
tests/models/multimodal/processing/test_common.py
+1 -1
@@ -89,7 +89,7 @@ def _test_processing_correctness(
89 mm_data = {
90 k:
91 [(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
92 - for _ in range(rng.randint(limit))]
92 + for _ in range(rng.randint(limit + 1))]
93 for k, limit in limit_mm_per_prompt.items()
94 }
95
tests/multimodal/utils.py
-3
@@ -17,10 +17,7 @@ def random_video(
17 min_wh: int,
18 max_wh: int,
19 ):
20 - # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
20 num_frames = rng.randint(min_frames, max_frames)
22 - num_frames = (num_frames // 2) * 2
23 -
21 w, h = rng.randint(min_wh, max_wh, size=(2, ))
22 return rng.randint(0, 255, size=(num_frames, w, h, 3), dtype=np.uint8)
23
vllm/model_executor/models/chatglm.py
+63 -97
@@ -4,8 +4,8 @@
4 # https://github.com/THUDM/CogAgent
5 """Inference-only CogAgent model compatible with THUDM weights."""
6 from argparse import Namespace
7 -from typing import (Iterable, List, Mapping, Optional, Sequence, Set, Tuple,
8 - TypedDict, Union)
7 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
8 + Union)
9
10 import torch
11 from torch import nn
@@ -19,7 +19,6 @@ from transformers.tokenization_utils_base import TextInput
19 from vllm.attention import Attention, AttentionMetadata
20 from vllm.config import CacheConfig, VllmConfig
21 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
22 -from vllm.logger import init_logger
22 from vllm.model_executor.layers.activation import SiluAndMul
23 from vllm.model_executor.layers.layernorm import RMSNorm
24 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -37,12 +36,10 @@ from vllm.model_executor.models.module_mapping import MultiModelKeys
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.multimodal import MULTIMODAL_REGISTRY
38 from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
40 -from vllm.multimodal.parse import ImageSize, MultiModalDataItems
39 +from vllm.multimodal.parse import MultiModalDataItems
40 from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 BaseProcessingInfo, BatchFeature,
43 - BoundPromptReplacement,
42 MultiModalFieldConfig,
45 - PlaceholderFeaturesInfo,
43 PromptReplacement)
44 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
45 from vllm.sequence import IntermediateTensors
@@ -53,39 +50,6 @@ from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
50 make_empty_intermediate_tensors_factory, make_layers,
51 maybe_prefix, merge_multimodal_embeddings)
52
56 -logger = init_logger(__name__)
57 -
58 -IMAGE_TOKEN_ID = 151329
59 -
60 -
61 -def build_normalization_transform(image_size: int) -> transforms.Compose:
62 - """
63 - Build a normalization transform which can be applied to one or
64 - more input images from which we want to extract visual features.
65 -
66 - Args:
67 - image_size: size of the image to be processed for visual embeddings.
68 -
69 - Returns:
70 - Callable transform for normalizing and resizing one RGB image.
71 - """
72 -
73 - return transforms.Compose([
74 - transforms.Resize(
75 - (image_size, image_size),
76 - interpolation=InterpolationMode.BICUBIC,
77 - ),
78 - transforms.ToTensor(),
79 - transforms.Normalize(
80 - (0.48145466, 0.4578275, 0.40821073),
81 - (0.26862954, 0.26130258, 0.27577711),
82 - ),
83 - ])
84 -
85 -
86 -def calculate_image_placeholder(vision_config):
87 - return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
88 -
53
54 class GLMImagePixelInputs(TypedDict):
55 pixel_values: torch.Tensor
@@ -109,9 +73,20 @@ class GLM4VProcessor:
73 self.config = config
74 self.tokenizer = tokenizer
75
112 - if hasattr(self.config, "vision_config"):
113 - self.image_transform = build_normalization_transform(
114 - config.vision_config["image_size"])
76 + if vision_config := getattr(config, "vision_config", None):
77 + image_size = vision_config["image_size"]
78 +
79 + self.image_transform = transforms.Compose([
80 + transforms.Resize(
81 + (image_size, image_size),
82 + interpolation=InterpolationMode.BICUBIC,
83 + ),
84 + transforms.ToTensor(),
85 + transforms.Normalize(
86 + mean=(0.48145466, 0.4578275, 0.40821073),
87 + std=(0.26862954, 0.26130258, 0.27577711),
88 + ),
89 + ])
90 else:
91 self.image_transform = None
92
@@ -150,9 +125,19 @@ class GLM4VProcessor:
125
126 class GLM4VProcessingInfo(BaseProcessingInfo):
127
153 - def __init__(self, ctx):
154 - super().__init__(ctx)
155 - self._pre_calculate()
128 + def get_tokenizer(self):
129 + tokenizer = self.ctx.tokenizer
130 + assert isinstance(tokenizer, PreTrainedTokenizer)
131 + return tokenizer
132 +
133 + def get_hf_config(self):
134 + return self.ctx.get_hf_config(ChatGLMConfig)
135 +
136 + def get_hf_processor(self) -> GLM4VProcessor:
137 + return GLM4VProcessor(
138 + self.get_hf_config(),
139 + self.get_tokenizer(),
140 + )
141
142 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
143 return {"image": 1}
@@ -162,27 +147,21 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
147 seq_len: int,
148 mm_counts: Mapping[str, int],
149 ) -> Mapping[str, int]:
150 + return {"image": self.get_num_image_feature_tokens()}
151
166 - return {"image": self.image_token_num + 2}
167 -
168 - def _pre_calculate(self):
152 + def get_num_image_tokens(self) -> int:
153 hf_config = self.get_hf_config()
170 - vision_config = hf_config.vision_config
171 - self.image_token_num = calculate_image_placeholder(vision_config)
172 - self.image_size = vision_config["image_size"]
154 + if not (vision_config := getattr(hf_config, "vision_config", None)):
155 + return 0
156
174 - def get_num_image_tokens(self) -> int:
175 - return self.image_token_num + 2
157 + image_size = vision_config["image_size"]
158 + patch_size = vision_config["patch_size"]
159 + grid_length = image_size // patch_size // 2
160 + return grid_length * grid_length
161
177 - def get_image_size(self) -> ImageSize:
178 -
179 - return ImageSize(height=self.image_size, width=self.image_size)
180 -
181 - def get_hf_processor(self) -> GLM4VProcessor:
182 - return GLM4VProcessor(
183 - self.get_hf_config(),
184 - self.get_tokenizer(),
185 - )
162 + def get_num_image_feature_tokens(self) -> int:
163 + # EVA2CLIPModel has embeddings for boi and eoi tokens as well
164 + return self.get_num_image_tokens() + 2
165
166
167 class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
@@ -192,8 +171,12 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
171 seq_len: int,
172 mm_counts: Mapping[str, int],
173 ) -> ProcessorInputs:
174 + hf_config = self.info.get_hf_config()
175 + if not (vision_config := getattr(hf_config, "vision_config", None)):
176 + return ProcessorInputs(prompt_text="", mm_data={})
177 +
178 + target_width = target_height = vision_config["image_size"]
179 num_images = mm_counts.get("image", 0)
196 - target_width, target_height = self.info.get_image_size()
180
181 mm_data = {
182 "image":
@@ -201,9 +184,11 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
184 height=target_height,
185 num_images=num_images)
186 }
204 - text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
187 +
188 + base_text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
189 +
190 return ProcessorInputs(
206 - prompt_text=text,
191 + prompt_text=base_text * num_images,
192 mm_data=mm_data,
193 )
194
@@ -223,47 +208,28 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
208 hf_processor_mm_kwargs: Mapping[str, object],
209 out_mm_kwargs: MultiModalKwargs,
210 ) -> list[PromptReplacement]:
211 + hf_config = self.info.get_hf_config()
212 + if not hasattr(hf_config, "vision_config"):
213 + return []
214 +
215 + boi_token_id = hf_config.boi_token_id
216 + image_token_id = hf_config.pad_token_id
217 + eoi_token_id = hf_config.eoi_token_id
218
219 def get_replacement(item_idx: int):
228 - image_tokens = self.info.image_token_num
229 - return [IMAGE_TOKEN_ID] * image_tokens
220 + num_image_tokens = self.info.get_num_image_tokens()
221 + image_tokens = [image_token_id] * num_image_tokens
222 +
223 + return [boi_token_id] + image_tokens + [eoi_token_id]
224
225 return [
226 PromptReplacement(
227 modality="image",
234 - target=[IMAGE_TOKEN_ID],
228 + target=[boi_token_id, image_token_id, eoi_token_id],
229 replacement=get_replacement,
230 ),
231 ]
232
239 - def _apply_prompt_replacements(
240 - self,
241 - token_ids: list[int],
242 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
243 - mm_item_counts: Mapping[str, int],
244 - ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
245 - token_ids, text, placeholders = super()._apply_prompt_replacements(
246 - token_ids=token_ids,
247 - mm_prompt_repls=mm_prompt_repls,
248 - mm_item_counts=mm_item_counts,
249 - )
250 - hf_config = self.info.get_hf_config()
251 - boi_token_id = hf_config.boi_token_id
252 - eoi_token_id = hf_config.eoi_token_id
253 - placeholders = {
254 - modality: [
255 - PlaceholderFeaturesInfo(
256 - modality=p.modality,
257 - item_idx=p.item_idx,
258 - start_idx=p.start_idx - 1,
259 - tokens=[boi_token_id] + p.tokens + [eoi_token_id],
260 - ) for p in ps
261 - ]
262 - for modality, ps in placeholders.items()
263 - }
264 -
265 - return token_ids, text, placeholders
266 -
233
234 class GLMAttention(nn.Module):
235
@@ -618,7 +584,7 @@ class ChatGLMModel(nn.Module):
584 multimodal_embeddings=multimodal_embeddings,
585 placeholder_token_id=[
586 self.config.boi_token_id,
621 - IMAGE_TOKEN_ID,
587 + self.config.pad_token_id,
588 self.config.eoi_token_id,
589 ],
590 )
vllm/model_executor/models/qwen.py
+46 -45
@@ -63,18 +63,6 @@ from .utils import (flatten_bn, is_pp_missing_parameter,
63
64 logger = init_logger(__name__)
65
66 -# NOTE: Qwen models have a few other special tags, e.g., ref, bbox, quad;
67 -# for the time being, these tags are not considered as special at encoding
68 -# time. This may change as VLLMs multimodal API changes in the future.
69 -IMG_START = "<img>"
70 -IMG_END = "</img>"
71 -IMG_PAD = "<imgpad>"
72 -# Image context is fixed at 256 for all images
73 -MAX_QWEN_IMG_TOKENS = 256
74 -# Image normalization params
75 -CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
76 -CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
77 -
66
67 class QwenImagePixelInputs(TypedDict):
68 type: Literal["pixel_values"]
@@ -622,25 +610,6 @@ class QWenModel(nn.Module):
610 return hidden_states
611
612
625 -def build_normalization_transform(image_size: int) -> transforms.Compose:
626 - """
627 - Build a normalization transform which can be applied to one or
628 - more input images from which we want to extract visual features.
629 -
630 - Args:
631 - image_size: size of the image to be processed for visual embeddings.
632 -
633 - Returns:
634 - Callable transform for normalizing and resizing one RGB image.
635 - """
636 - return transforms.Compose([
637 - transforms.Resize((image_size, image_size),
638 - interpolation=InterpolationMode.BICUBIC),
639 - transforms.ToTensor(),
640 - transforms.Normalize(mean=CLIP_MEAN, std=CLIP_STD),
641 - ])
642 -
643 -
613 @lru_cache(maxsize=1)
614 def _get_tokenizer_without_image_pad(
615 tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
@@ -716,16 +685,34 @@ class QWenVLProcessor:
685 self.config = config
686 self.tokenizer = tokenizer
687
719 - if hasattr(self.config, "visual"):
720 - self.image_transform = build_normalization_transform(
721 - config.visual["image_size"])
688 + if vision_config := getattr(self.config, "visual", None):
689 + image_size = vision_config["image_size"]
690 +
691 + self.image_transform = transforms.Compose([
692 + transforms.Resize(
693 + (image_size, image_size),
694 + interpolation=InterpolationMode.BICUBIC,
695 + ),
696 + transforms.ToTensor(),
697 + transforms.Normalize(
698 + mean=(0.48145466, 0.4578275, 0.40821073),
699 + std=(0.26862954, 0.26130258, 0.27577711),
700 + ),
701 + ])
702 else:
703 self.image_transform = None
704
725 - special_tokens: dict[str,
726 - int] = tokenizer.special_tokens # type: ignore
727 - self.img_start_id = special_tokens[IMG_START]
728 - self.img_end_id = special_tokens[IMG_END]
705 + @property
706 + def image_start_tag(self) -> str:
707 + return self.tokenizer.image_start_tag # type: ignore
708 +
709 + @property
710 + def image_end_tag(self) -> str:
711 + return self.tokenizer.image_end_tag # type: ignore
712 +
713 + @property
714 + def image_pad_tag(self) -> str:
715 + return self.tokenizer.image_pad_tag # type: ignore
716
717 def __call__(
718 self,
@@ -787,7 +774,14 @@ class QWenVLProcessingInfo(BaseProcessingInfo):
774 return {"image": self.get_num_image_tokens()}
775
776 def get_num_image_tokens(self) -> int:
790 - return MAX_QWEN_IMG_TOKENS
777 + hf_config = self.get_hf_config()
778 + if not (vision_config := getattr(hf_config, "visual", None)):
779 + return 0
780 +
781 + image_size = vision_config["image_size"]
782 + patch_size = vision_config["patch_size"]
783 + grid_length = image_size // patch_size // 2
784 + return grid_length * grid_length
785
786
787 class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
@@ -798,10 +792,12 @@ class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
792 mm_counts: Mapping[str, int],
793 ) -> ProcessorInputs:
794 hf_config = self.info.get_hf_config()
801 - if not hasattr(hf_config, "visual"):
795 + if not (vision_config := getattr(hf_config, "visual", None)):
796 return ProcessorInputs(prompt_text="", mm_data={})
797
804 - vision_config = hf_config.visual
798 + processor = self.info.get_hf_processor()
799 + img_start = processor.image_start_tag
800 + img_end = processor.image_end_tag
801
802 target_width = target_height = vision_config["image_size"]
803 num_images = mm_counts.get("image", 0)
@@ -814,7 +810,7 @@ class QWenVLDummyInputsBuilder(BaseDummyInputsBuilder[QWenVLProcessingInfo]):
810 }
811
812 return ProcessorInputs(
817 - prompt_text="".join(f"Picture {i}: {IMG_START}{IMG_END}\n"
813 + prompt_text="".join(f"Picture {i}: {img_start}{img_end}\n"
814 for i in range(1, num_images + 1)),
815 mm_data=mm_data,
816 )
@@ -869,13 +865,18 @@ class QWenVLMultiModalProcessor(BaseMultiModalProcessor[QWenVLProcessingInfo]):
865 hf_processor_mm_kwargs: Mapping[str, object],
866 out_mm_kwargs: MultiModalKwargs,
867 ) -> list[PromptReplacement]:
868 + hf_config = self.info.get_hf_config()
869 + if not hasattr(hf_config, "visual"):
870 + return []
871 +
872 tokenizer = self.info.get_tokenizer()
873 special_tokens: dict[str,
874 int] = tokenizer.special_tokens # type: ignore
875
876 - img_start_id = special_tokens[IMG_START]
877 - img_end_id = special_tokens[IMG_END]
878 - img_pad_id = special_tokens[IMG_PAD]
876 + processor = self.info.get_hf_processor()
877 + img_start_id = special_tokens[processor.image_start_tag]
878 + img_end_id = special_tokens[processor.image_end_tag]
879 + img_pad_id = special_tokens[processor.image_pad_tag]
880
881 num_image_tokens = self.info.get_num_image_tokens()
882 image_tokens = [img_pad_id] * num_image_tokens
vllm/model_executor/models/qwen2_vl.py
+3 -7
@@ -885,14 +885,10 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
885 max_image_tokens = self.get_max_image_tokens() * max_images
886 max_total_frames = self._get_max_video_frames(seq_len -
887 max_image_tokens)
888 - num_frames = min(max(max_total_frames // max(max_videos, 1), 1),
889 - _MAX_FRAMES_PER_VIDEO)
888 + max_frames_per_video = min(max_total_frames // max(max_videos, 1),
889 + _MAX_FRAMES_PER_VIDEO)
890
891 - # Temporary workaround for https://github.com/huggingface/transformers/issues/35412
892 - if num_frames > 1 and num_frames % 2 == 1:
893 - num_frames += 1
894 -
895 - return num_frames
891 + return max(max_frames_per_video, 1)
892
893 def get_max_video_tokens(self, seq_len: int) -> int:
894 target_width, target_height = self.get_image_size_with_most_features()