A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[VLM] Merged multi-modal processor for GLM4V (#12449)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Feb 9, 2025 at 04:32 UTC
86222a3dab50b66bb0bff17a94b629aa59c3ed57
4 files changed
+222
-167
docs/source/models/supported_models.md
+1
-1
@@ -719,7 +719,7 @@ See [this page](#generative-models) for more information on how to use generativ
719
* `THUDM/glm-4v-9b` etc.
720
* ✅︎
721
* ✅︎
722
- *
722
+ * ✅︎
723
- * `H2OVLChatModel`
724
* H2OVL
725
* T + I<sup>E+</sup>
examples/offline_inference/vision_language.py
+3
-1
@@ -106,7 +106,9 @@ def run_glm4v(question: str, modality: str):
106
trust_remote_code=True,
107
enforce_eager=True,
108
disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
109
- prompt = question
109
+ prompt = f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
110
+ {question}<|assistant|>"
111
+
112
stop_token_ids = [151329, 151336, 151338]
113
return llm, prompt, stop_token_ids
114
tests/models/multimodal/processing/test_common.py
+1
@@ -147,6 +147,7 @@ def _test_processing_correctness(
147
"facebook/chameleon-7b",
148
"deepseek-ai/deepseek-vl2-tiny",
149
"adept/fuyu-8b",
150
+ "THUDM/glm-4v-9b",
151
"h2oai/h2ovl-mississippi-800m",
152
"OpenGVLab/InternVL2-1B",
153
"HuggingFaceM4/Idefics3-8B-Llama3",
vllm/model_executor/models/chatglm.py
+217
-165
@@ -4,20 +4,21 @@
4
# https://github.com/THUDM/CogAgent
5
"""Inference-only CogAgent model compatible with THUDM weights."""
6
from argparse import Namespace
7
-from array import array
8
-from typing import (Dict, Iterable, List, Mapping, Optional, Set, Tuple,
9
- TypedDict)
7
+from typing import (Iterable, List, Mapping, Optional, Sequence, Set, Tuple,
8
+ TypedDict, Union)
9
10
import torch
12
-from PIL import Image
11
from torch import nn
12
from torch.nn import LayerNorm
13
+from torchvision import transforms
14
+from torchvision.transforms import InterpolationMode
15
+from transformers import PreTrainedTokenizer, TensorType
16
+from transformers.image_utils import ImageInput
17
+from transformers.tokenization_utils_base import TextInput
18
19
from vllm.attention import Attention, AttentionMetadata
20
from vllm.config import CacheConfig, VllmConfig
21
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
19
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20
- InputContext, token_inputs)
22
from vllm.logger import init_logger
23
from vllm.model_executor.layers.activation import SiluAndMul
24
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -35,73 +36,55 @@ from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
36
from vllm.model_executor.models.module_mapping import MultiModelKeys
37
from vllm.model_executor.sampling_metadata import SamplingMetadata
38
from vllm.multimodal import MULTIMODAL_REGISTRY
38
-from vllm.multimodal.inputs import (ModalityData, MultiModalKwargs,
39
- NestedTensors)
40
-from vllm.multimodal.utils import cached_get_tokenizer
41
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42
- SequenceData)
39
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
40
+from vllm.multimodal.parse import ImageSize, MultiModalDataItems
41
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
42
+ BaseProcessingInfo, BatchFeature,
43
+ BoundPromptReplacement,
44
+ MultiModalFieldConfig,
45
+ PlaceholderFeaturesInfo,
46
+ PromptReplacement)
47
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
48
+from vllm.sequence import IntermediateTensors
49
from vllm.transformers_utils.configs import ChatGLMConfig
50
51
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
52
from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
53
make_empty_intermediate_tensors_factory, make_layers,
48
- maybe_prefix)
54
+ maybe_prefix, merge_multimodal_embeddings)
55
56
logger = init_logger(__name__)
57
58
+IMAGE_TOKEN_ID = 151329
59
53
-def calculate_image_placeholder(vision_config):
54
- return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
60
61
+def build_normalization_transform(image_size: int) -> transforms.Compose:
62
+ """
63
+ Build a normalization transform which can be applied to one or
64
+ more input images from which we want to extract visual features.
65
+
66
+ Args:
67
+ image_size: size of the image to be processed for visual embeddings.
68
+
69
+ Returns:
70
+ Callable transform for normalizing and resizing one RGB image.
71
+ """
72
57
-def mm_input_mapper_for_glmv(
58
- ctx: InputContext,
59
- data: ModalityData[object],
60
-) -> Dict:
61
- model_config = ctx.model_config
62
- tokenizer = cached_get_tokenizer(
63
- model_config.tokenizer,
64
- trust_remote_code=model_config.trust_remote_code)
65
- if tokenizer is None:
66
- raise RuntimeError("No HuggingFace processor is available "
67
- "to process the image object")
68
- try:
69
- raw_batch_data = tokenizer.apply_chat_template(
70
- conversation=[{
71
- "role": "user",
72
- "image": data
73
- }],
74
- add_generation_prompt=True,
75
- tokenize=True,
76
- return_tensors="pt",
77
- return_dict=True).data
78
- except Exception:
79
- logger.error("Failed to process image (%s)", data)
80
- raise
81
- pixel_values = raw_batch_data['images']
82
-
83
- return MultiModalKwargs({'pixel_values': pixel_values})
84
-
85
-
86
-def merge_glm_vision_embeddings(
87
- input_ids: torch.Tensor,
88
- inputs_embeds: torch.Tensor,
89
- vision_embeddings: torch.Tensor,
90
- boi_token_id: int,
91
- eoi_token_id: int,
92
-) -> torch.Tensor:
93
-
94
- boi_positions = (input_ids == boi_token_id).nonzero(as_tuple=True)[0]
95
- eoi_positions = (input_ids == eoi_token_id).nonzero(as_tuple=True)[0]
96
-
97
- mask = torch.zeros_like(input_ids, dtype=torch.bool)
98
-
99
- for boi_pos, eoi_pos in zip(boi_positions, eoi_positions):
100
- assert boi_pos < eoi_pos
101
- mask[boi_pos:eoi_pos + 1] = True
102
- inputs_embeds[mask] = vision_embeddings.view(-1,
103
- vision_embeddings.shape[-1])
104
- return inputs_embeds
73
+ return transforms.Compose([
74
+ transforms.Resize(
75
+ (image_size, image_size),
76
+ interpolation=InterpolationMode.BICUBIC,
77
+ ),
78
+ transforms.ToTensor(),
79
+ transforms.Normalize(
80
+ (0.48145466, 0.4578275, 0.40821073),
81
+ (0.26862954, 0.26130258, 0.27577711),
82
+ ),
83
+ ])
84
+
85
+
86
+def calculate_image_placeholder(vision_config):
87
+ return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
88
89
90
class GLMImagePixelInputs(TypedDict):
@@ -109,120 +92,177 @@ class GLMImagePixelInputs(TypedDict):
92
"""Shape: `(batch_size, num_channels, height, width)`"""
93
94
112
-def get_max_glmv_image_tokens(ctx: InputContext):
113
- hf_config = ctx.get_hf_config(ChatGLMConfig)
95
+class GLM4VProcessor:
96
+ """
97
+ This model doesn't define its own HF processor,
98
+ so we implement our own one here.
99
115
- vision_config = getattr(hf_config, 'vision_config', None)
116
- if vision_config is None:
117
- return 1
118
- elif isinstance(vision_config, dict):
119
- return calculate_image_placeholder(vision_config)
100
+ """
101
121
- msg = f"Unsupported vision config: {type(vision_config)}"
122
- raise NotImplementedError(msg)
102
+ def __init__(
103
+ self,
104
+ config: ChatGLMConfig,
105
+ tokenizer: PreTrainedTokenizer,
106
+ ) -> None:
107
+ super().__init__()
108
109
+ self.config = config
110
+ self.tokenizer = tokenizer
111
125
-def dummy_data_for_glmv(ctx: InputContext, seq_len: int,
126
- mm_counts: Mapping[str, int]) -> DummyData:
127
- hf_config = ctx.get_hf_config(ChatGLMConfig)
128
- vision_config = getattr(hf_config, 'vision_config', None)
112
+ if hasattr(self.config, "vision_config"):
113
+ self.image_transform = build_normalization_transform(
114
+ config.vision_config["image_size"])
115
+ else:
116
+ self.image_transform = None
117
130
- if vision_config is None:
131
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len)
132
- seq_data = SequenceData(token_ids)
133
- return DummyData(seq_data, None)
134
- elif isinstance(vision_config, dict):
135
- image_size = vision_config["image_size"]
136
- image_placeholder_length = calculate_image_placeholder(vision_config)
137
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [hf_config.boi_token_id] +
138
- [0] * image_placeholder_length +
139
- [hf_config.eoi_token_id])
140
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
141
- [0] * (seq_len - image_placeholder_length - 2))
142
- seq_data = SequenceData(token_ids)
118
+ def __call__(
119
+ self,
120
+ text: Optional[Union[TextInput, list[TextInput]]] = None,
121
+ images: Optional[Union[ImageInput, list[ImageInput]]] = None,
122
+ return_tensors: Optional[Union[str, TensorType]] = None,
123
+ ) -> BatchFeature:
124
+ if text is None:
125
+ text = []
126
+ if not isinstance(text, list):
127
+ text = [text]
128
+ if images is None:
129
+ images = []
130
+ if not isinstance(images, list):
131
+ images = [images]
132
+ text_inputs = self.tokenizer(text)
133
+ if len(images) == 0:
134
+ image_inputs = {}
135
+ else:
136
+ if self.image_transform is None:
137
+ raise ValueError("This model does not support image inputs")
138
+
139
+ pixel_values = [self.image_transform(image) for image in images]
140
+ image_inputs = {"pixel_values": torch.stack(pixel_values)}
141
+
142
+ return BatchFeature(
143
+ {
144
+ **text_inputs,
145
+ **image_inputs,
146
+ },
147
+ tensor_type=return_tensors,
148
+ )
149
144
- mm_data = {
145
- "image": Image.new("RGB", (image_size, image_size), color=0)
146
- }
150
148
- return DummyData(seq_data, mm_data)
151
+class GLM4VProcessingInfo(BaseProcessingInfo):
152
150
- msg = f"Unsupported vision config: {type(vision_config)}"
151
- raise NotImplementedError(msg)
153
+ def __init__(self, ctx):
154
+ super().__init__(ctx)
155
+ self._pre_calculate()
156
157
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
158
+ return {"image": 1}
159
154
-def find_all_positions(input_ids: List[int], target: int) -> List[int]:
155
- return [index for index, value in enumerate(input_ids) if value == target]
160
+ def get_mm_max_tokens_per_item(
161
+ self,
162
+ seq_len: int,
163
+ mm_counts: Mapping[str, int],
164
+ ) -> Mapping[str, int]:
165
166
+ return {"image": self.image_token_num + 2}
167
158
-def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
159
- multi_modal_data = inputs.get("multi_modal_data")
160
- if multi_modal_data is None or "image" not in multi_modal_data:
161
- return inputs
168
+ def _pre_calculate(self):
169
+ hf_config = self.get_hf_config()
170
+ vision_config = hf_config.vision_config
171
+ self.image_token_num = calculate_image_placeholder(vision_config)
172
+ self.image_size = vision_config["image_size"]
173
163
- hf_config = ctx.get_hf_config(ChatGLMConfig)
164
- vision_config = getattr(hf_config, 'vision_config', None)
174
+ def get_num_image_tokens(self) -> int:
175
+ return self.image_token_num + 2
176
166
- if vision_config is None:
167
- return inputs
168
- elif isinstance(vision_config, dict):
169
- image_placeholder_length = calculate_image_placeholder(vision_config)
170
- else:
171
- msg = f"Unsupported vision config: {type(vision_config)}"
172
- raise NotImplementedError(msg)
177
+ def get_image_size(self) -> ImageSize:
178
174
- input_ids = inputs["prompt_token_ids"]
179
+ return ImageSize(height=self.image_size, width=self.image_size)
180
176
- tokenizer = cached_get_tokenizer(
177
- ctx.model_config.model,
178
- trust_remote_code=ctx.model_config.trust_remote_code)
181
+ def get_hf_processor(self) -> GLM4VProcessor:
182
+ return GLM4VProcessor(
183
+ self.get_hf_config(),
184
+ self.get_tokenizer(),
185
+ )
186
180
- try:
181
- raw_batch_data = tokenizer.apply_chat_template(
182
- conversation=[{
183
- "role": "user",
184
- "image": multi_modal_data["image"],
185
- "content": inputs['prompt'],
186
- }],
187
- add_generation_prompt=True,
188
- tokenize=True,
189
- return_tensors="pt",
190
- return_dict=True,
191
- ).data
192
- except Exception:
193
- logger.error("Failed to process content (%s)", inputs['prompt'])
194
- raise
195
- input_ids = raw_batch_data['input_ids'][0].tolist()
187
197
- boi_token_id = hf_config.boi_token_id
198
- eoi_token_id = hf_config.eoi_token_id
199
- boi_positions = find_all_positions(input_ids, boi_token_id)
200
- eoi_positions = find_all_positions(input_ids, eoi_token_id)
188
+class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
189
202
- assert len(boi_positions) == len(eoi_positions)
190
+ def get_dummy_processor_inputs(
191
+ self,
192
+ seq_len: int,
193
+ mm_counts: Mapping[str, int],
194
+ ) -> ProcessorInputs:
195
+ num_images = mm_counts.get("image", 0)
196
+ target_width, target_height = self.info.get_image_size()
197
204
- new_input_ids = []
205
- final_processed_position = 0
198
+ mm_data = {
199
+ "image":
200
+ self._get_dummy_images(width=target_width,
201
+ height=target_height,
202
+ num_images=num_images)
203
+ }
204
+ text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
205
+ return ProcessorInputs(
206
+ prompt_text=text,
207
+ mm_data=mm_data,
208
+ )
209
207
- for boi_position, eoi_position in zip(boi_positions, eoi_positions):
208
- assert boi_position < eoi_position
209
- new_input_ids.extend(input_ids[final_processed_position:boi_position +
210
- 1])
211
- new_input_ids.extend([input_ids[boi_position + 1]] *
212
- image_placeholder_length)
213
- final_processed_position = eoi_position
210
215
- new_input_ids.extend(input_ids[final_processed_position:])
211
+class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
212
+
213
+ def _get_mm_fields_config(
214
+ self,
215
+ hf_inputs: BatchFeature,
216
+ hf_processor_mm_kwargs: Mapping[str, object],
217
+ ) -> Mapping[str, MultiModalFieldConfig]:
218
+ return dict(pixel_values=MultiModalFieldConfig.batched("image"))
219
+
220
+ def _get_prompt_replacements(
221
+ self,
222
+ mm_items: MultiModalDataItems,
223
+ hf_processor_mm_kwargs: Mapping[str, object],
224
+ out_mm_kwargs: MultiModalKwargs,
225
+ ) -> list[PromptReplacement]:
226
+
227
+ def get_replacement(item_idx: int):
228
+ image_tokens = self.info.image_token_num
229
+ return [IMAGE_TOKEN_ID] * image_tokens
230
+
231
+ return [
232
+ PromptReplacement(
233
+ modality="image",
234
+ target=[IMAGE_TOKEN_ID],
235
+ replacement=get_replacement,
236
+ ),
237
+ ]
238
217
- prompt = inputs.get("prompt")
218
- if prompt is None:
219
- prompt = tokenizer.decode(new_input_ids)
239
+ def _apply_prompt_replacements(
240
+ self,
241
+ token_ids: list[int],
242
+ mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
243
+ mm_item_counts: Mapping[str, int],
244
+ ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
245
+ token_ids, text, placeholders = super()._apply_prompt_replacements(
246
+ token_ids=token_ids,
247
+ mm_prompt_repls=mm_prompt_repls,
248
+ mm_item_counts=mm_item_counts,
249
+ )
250
+ hf_config = self.info.get_hf_config()
251
+ boi_token_id = hf_config.boi_token_id
252
+ eoi_token_id = hf_config.eoi_token_id
253
+ placeholders = {
254
+ modality: [
255
+ PlaceholderFeaturesInfo(
256
+ modality=p.modality,
257
+ item_idx=p.item_idx,
258
+ start_idx=p.start_idx - 1,
259
+ tokens=[boi_token_id] + p.tokens + [eoi_token_id],
260
+ ) for p in ps
261
+ ]
262
+ for modality, ps in placeholders.items()
263
+ }
264
221
- return token_inputs(
222
- prompt_token_ids=new_input_ids,
223
- prompt=prompt,
224
- multi_modal_data=multi_modal_data,
225
- )
265
+ return token_ids, text, placeholders
266
267
268
class GLMAttention(nn.Module):
@@ -572,12 +612,16 @@ class ChatGLMModel(nn.Module):
612
) -> torch.Tensor:
613
inputs_embeds = self.embedding(input_ids)
614
if multimodal_embeddings is not None:
575
- inputs_embeds = merge_glm_vision_embeddings(
615
+ inputs_embeds = merge_multimodal_embeddings(
616
input_ids=input_ids,
617
inputs_embeds=inputs_embeds,
578
- vision_embeddings=multimodal_embeddings,
579
- boi_token_id=self.config.boi_token_id,
580
- eoi_token_id=self.config.eoi_token_id)
618
+ multimodal_embeddings=multimodal_embeddings,
619
+ placeholder_token_id=[
620
+ self.config.boi_token_id,
621
+ IMAGE_TOKEN_ID,
622
+ self.config.eoi_token_id,
623
+ ],
624
+ )
625
return inputs_embeds
626
627
def forward(
@@ -593,14 +637,12 @@ class ChatGLMModel(nn.Module):
637
638
# NOTE: In v1, inputs_embeds is always generated at model runner, this
639
# condition is for v0 compatibility.
596
- if intermediate_tensors is None and inputs_embeds is None:
640
+ if intermediate_tensors is not None:
641
+ inputs_embeds = intermediate_tensors["hidden_states"]
642
+ elif inputs_embeds is None:
643
vision_embeddings = self.get_multimodal_embeddings(**kwargs)
644
inputs_embeds = self.get_input_embeddings(input_ids,
645
vision_embeddings)
600
- input_ids = None
601
- else:
602
- inputs_embeds = intermediate_tensors["hidden_states"]
603
-
646
# Run encoder.
647
hidden_states = self.encoder(
648
hidden_states=inputs_embeds,
@@ -763,11 +805,21 @@ class ChatGLMV(ChatGLMBaseModel, SupportsMultiModal):
805
connector="transformer.vision.linear_proj",
806
tower_model="transformer.vision.transformer")
807
808
+ def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
809
+ return self.transformer.get_multimodal_embeddings(**kwargs)
810
+
811
+ def get_input_embeddings(
812
+ self,
813
+ input_ids: torch.Tensor,
814
+ multimodal_embeddings: Optional[NestedTensors] = None,
815
+ ) -> torch.Tensor:
816
+ return self.transformer.get_input_embeddings(input_ids,
817
+ multimodal_embeddings)
818
+
819
767
-@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
768
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
769
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
770
-@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
820
+@MULTIMODAL_REGISTRY.register_processor(GLM4VMultiModalProcessor,
821
+ info=GLM4VProcessingInfo,
822
+ dummy_inputs=GLM4VDummyInputsBuilder)
823
class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
824
SupportsMultiModal):
825
# Ensure that the LoRA support check passes when the class is not