@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 86222a3dab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[VLM] Merged multi-modal processor for GLM4V (#12449)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Feb 9, 2025 at 04:32 UTC 86222a3dab50b66bb0bff17a94b629aa59c3ed57
4 files changed +222 -167
docs/source/models/supported_models.md
+1 -1
@@ -719,7 +719,7 @@ See [this page](#generative-models) for more information on how to use generativ
719 * `THUDM/glm-4v-9b` etc.
720 * ✅︎
721 * ✅︎
722 - *
722 + * ✅︎
723 - * `H2OVLChatModel`
724 * H2OVL
725 * T + I<sup>E+</sup>
examples/offline_inference/vision_language.py
+3 -1
@@ -106,7 +106,9 @@ def run_glm4v(question: str, modality: str):
106 trust_remote_code=True,
107 enforce_eager=True,
108 disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache)
109 - prompt = question
109 + prompt = f"<|user|>\n<|begin_of_image|><|endoftext|><|end_of_image|>\
110 + {question}<|assistant|>"
111 +
112 stop_token_ids = [151329, 151336, 151338]
113 return llm, prompt, stop_token_ids
114
tests/models/multimodal/processing/test_common.py
+1
@@ -147,6 +147,7 @@ def _test_processing_correctness(
147 "facebook/chameleon-7b",
148 "deepseek-ai/deepseek-vl2-tiny",
149 "adept/fuyu-8b",
150 + "THUDM/glm-4v-9b",
151 "h2oai/h2ovl-mississippi-800m",
152 "OpenGVLab/InternVL2-1B",
153 "HuggingFaceM4/Idefics3-8B-Llama3",
vllm/model_executor/models/chatglm.py
+217 -165
@@ -4,20 +4,21 @@
4 # https://github.com/THUDM/CogAgent
5 """Inference-only CogAgent model compatible with THUDM weights."""
6 from argparse import Namespace
7 -from array import array
8 -from typing import (Dict, Iterable, List, Mapping, Optional, Set, Tuple,
9 - TypedDict)
7 +from typing import (Iterable, List, Mapping, Optional, Sequence, Set, Tuple,
8 + TypedDict, Union)
9
10 import torch
12 -from PIL import Image
11 from torch import nn
12 from torch.nn import LayerNorm
13 +from torchvision import transforms
14 +from torchvision.transforms import InterpolationMode
15 +from transformers import PreTrainedTokenizer, TensorType
16 +from transformers.image_utils import ImageInput
17 +from transformers.tokenization_utils_base import TextInput
18
19 from vllm.attention import Attention, AttentionMetadata
20 from vllm.config import CacheConfig, VllmConfig
21 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
19 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20 - InputContext, token_inputs)
22 from vllm.logger import init_logger
23 from vllm.model_executor.layers.activation import SiluAndMul
24 from vllm.model_executor.layers.layernorm import RMSNorm
@@ -35,73 +36,55 @@ from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
36 from vllm.model_executor.models.module_mapping import MultiModelKeys
37 from vllm.model_executor.sampling_metadata import SamplingMetadata
38 from vllm.multimodal import MULTIMODAL_REGISTRY
38 -from vllm.multimodal.inputs import (ModalityData, MultiModalKwargs,
39 - NestedTensors)
40 -from vllm.multimodal.utils import cached_get_tokenizer
41 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
42 - SequenceData)
39 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
40 +from vllm.multimodal.parse import ImageSize, MultiModalDataItems
41 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
42 + BaseProcessingInfo, BatchFeature,
43 + BoundPromptReplacement,
44 + MultiModalFieldConfig,
45 + PlaceholderFeaturesInfo,
46 + PromptReplacement)
47 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
48 +from vllm.sequence import IntermediateTensors
49 from vllm.transformers_utils.configs import ChatGLMConfig
50
51 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
52 from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
53 make_empty_intermediate_tensors_factory, make_layers,
48 - maybe_prefix)
54 + maybe_prefix, merge_multimodal_embeddings)
55
56 logger = init_logger(__name__)
57
58 +IMAGE_TOKEN_ID = 151329
59
53 -def calculate_image_placeholder(vision_config):
54 - return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
60
61 +def build_normalization_transform(image_size: int) -> transforms.Compose:
62 + """
63 + Build a normalization transform which can be applied to one or
64 + more input images from which we want to extract visual features.
65 +
66 + Args:
67 + image_size: size of the image to be processed for visual embeddings.
68 +
69 + Returns:
70 + Callable transform for normalizing and resizing one RGB image.
71 + """
72
57 -def mm_input_mapper_for_glmv(
58 - ctx: InputContext,
59 - data: ModalityData[object],
60 -) -> Dict:
61 - model_config = ctx.model_config
62 - tokenizer = cached_get_tokenizer(
63 - model_config.tokenizer,
64 - trust_remote_code=model_config.trust_remote_code)
65 - if tokenizer is None:
66 - raise RuntimeError("No HuggingFace processor is available "
67 - "to process the image object")
68 - try:
69 - raw_batch_data = tokenizer.apply_chat_template(
70 - conversation=[{
71 - "role": "user",
72 - "image": data
73 - }],
74 - add_generation_prompt=True,
75 - tokenize=True,
76 - return_tensors="pt",
77 - return_dict=True).data
78 - except Exception:
79 - logger.error("Failed to process image (%s)", data)
80 - raise
81 - pixel_values = raw_batch_data['images']
82 -
83 - return MultiModalKwargs({'pixel_values': pixel_values})
84 -
85 -
86 -def merge_glm_vision_embeddings(
87 - input_ids: torch.Tensor,
88 - inputs_embeds: torch.Tensor,
89 - vision_embeddings: torch.Tensor,
90 - boi_token_id: int,
91 - eoi_token_id: int,
92 -) -> torch.Tensor:
93 -
94 - boi_positions = (input_ids == boi_token_id).nonzero(as_tuple=True)[0]
95 - eoi_positions = (input_ids == eoi_token_id).nonzero(as_tuple=True)[0]
96 -
97 - mask = torch.zeros_like(input_ids, dtype=torch.bool)
98 -
99 - for boi_pos, eoi_pos in zip(boi_positions, eoi_positions):
100 - assert boi_pos < eoi_pos
101 - mask[boi_pos:eoi_pos + 1] = True
102 - inputs_embeds[mask] = vision_embeddings.view(-1,
103 - vision_embeddings.shape[-1])
104 - return inputs_embeds
73 + return transforms.Compose([
74 + transforms.Resize(
75 + (image_size, image_size),
76 + interpolation=InterpolationMode.BICUBIC,
77 + ),
78 + transforms.ToTensor(),
79 + transforms.Normalize(
80 + (0.48145466, 0.4578275, 0.40821073),
81 + (0.26862954, 0.26130258, 0.27577711),
82 + ),
83 + ])
84 +
85 +
86 +def calculate_image_placeholder(vision_config):
87 + return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
88
89
90 class GLMImagePixelInputs(TypedDict):
@@ -109,120 +92,177 @@ class GLMImagePixelInputs(TypedDict):
92 """Shape: `(batch_size, num_channels, height, width)`"""
93
94
112 -def get_max_glmv_image_tokens(ctx: InputContext):
113 - hf_config = ctx.get_hf_config(ChatGLMConfig)
95 +class GLM4VProcessor:
96 + """
97 + This model doesn't define its own HF processor,
98 + so we implement our own one here.
99
115 - vision_config = getattr(hf_config, 'vision_config', None)
116 - if vision_config is None:
117 - return 1
118 - elif isinstance(vision_config, dict):
119 - return calculate_image_placeholder(vision_config)
100 + """
101
121 - msg = f"Unsupported vision config: {type(vision_config)}"
122 - raise NotImplementedError(msg)
102 + def __init__(
103 + self,
104 + config: ChatGLMConfig,
105 + tokenizer: PreTrainedTokenizer,
106 + ) -> None:
107 + super().__init__()
108
109 + self.config = config
110 + self.tokenizer = tokenizer
111
125 -def dummy_data_for_glmv(ctx: InputContext, seq_len: int,
126 - mm_counts: Mapping[str, int]) -> DummyData:
127 - hf_config = ctx.get_hf_config(ChatGLMConfig)
128 - vision_config = getattr(hf_config, 'vision_config', None)
112 + if hasattr(self.config, "vision_config"):
113 + self.image_transform = build_normalization_transform(
114 + config.vision_config["image_size"])
115 + else:
116 + self.image_transform = None
117
130 - if vision_config is None:
131 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len)
132 - seq_data = SequenceData(token_ids)
133 - return DummyData(seq_data, None)
134 - elif isinstance(vision_config, dict):
135 - image_size = vision_config["image_size"]
136 - image_placeholder_length = calculate_image_placeholder(vision_config)
137 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [hf_config.boi_token_id] +
138 - [0] * image_placeholder_length +
139 - [hf_config.eoi_token_id])
140 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
141 - [0] * (seq_len - image_placeholder_length - 2))
142 - seq_data = SequenceData(token_ids)
118 + def __call__(
119 + self,
120 + text: Optional[Union[TextInput, list[TextInput]]] = None,
121 + images: Optional[Union[ImageInput, list[ImageInput]]] = None,
122 + return_tensors: Optional[Union[str, TensorType]] = None,
123 + ) -> BatchFeature:
124 + if text is None:
125 + text = []
126 + if not isinstance(text, list):
127 + text = [text]
128 + if images is None:
129 + images = []
130 + if not isinstance(images, list):
131 + images = [images]
132 + text_inputs = self.tokenizer(text)
133 + if len(images) == 0:
134 + image_inputs = {}
135 + else:
136 + if self.image_transform is None:
137 + raise ValueError("This model does not support image inputs")
138 +
139 + pixel_values = [self.image_transform(image) for image in images]
140 + image_inputs = {"pixel_values": torch.stack(pixel_values)}
141 +
142 + return BatchFeature(
143 + {
144 + **text_inputs,
145 + **image_inputs,
146 + },
147 + tensor_type=return_tensors,
148 + )
149
144 - mm_data = {
145 - "image": Image.new("RGB", (image_size, image_size), color=0)
146 - }
150
148 - return DummyData(seq_data, mm_data)
151 +class GLM4VProcessingInfo(BaseProcessingInfo):
152
150 - msg = f"Unsupported vision config: {type(vision_config)}"
151 - raise NotImplementedError(msg)
153 + def __init__(self, ctx):
154 + super().__init__(ctx)
155 + self._pre_calculate()
156
157 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
158 + return {"image": 1}
159
154 -def find_all_positions(input_ids: List[int], target: int) -> List[int]:
155 - return [index for index, value in enumerate(input_ids) if value == target]
160 + def get_mm_max_tokens_per_item(
161 + self,
162 + seq_len: int,
163 + mm_counts: Mapping[str, int],
164 + ) -> Mapping[str, int]:
165
166 + return {"image": self.image_token_num + 2}
167
158 -def input_processor_for_glmv(ctx: InputContext, inputs: DecoderOnlyInputs):
159 - multi_modal_data = inputs.get("multi_modal_data")
160 - if multi_modal_data is None or "image" not in multi_modal_data:
161 - return inputs
168 + def _pre_calculate(self):
169 + hf_config = self.get_hf_config()
170 + vision_config = hf_config.vision_config
171 + self.image_token_num = calculate_image_placeholder(vision_config)
172 + self.image_size = vision_config["image_size"]
173
163 - hf_config = ctx.get_hf_config(ChatGLMConfig)
164 - vision_config = getattr(hf_config, 'vision_config', None)
174 + def get_num_image_tokens(self) -> int:
175 + return self.image_token_num + 2
176
166 - if vision_config is None:
167 - return inputs
168 - elif isinstance(vision_config, dict):
169 - image_placeholder_length = calculate_image_placeholder(vision_config)
170 - else:
171 - msg = f"Unsupported vision config: {type(vision_config)}"
172 - raise NotImplementedError(msg)
177 + def get_image_size(self) -> ImageSize:
178
174 - input_ids = inputs["prompt_token_ids"]
179 + return ImageSize(height=self.image_size, width=self.image_size)
180
176 - tokenizer = cached_get_tokenizer(
177 - ctx.model_config.model,
178 - trust_remote_code=ctx.model_config.trust_remote_code)
181 + def get_hf_processor(self) -> GLM4VProcessor:
182 + return GLM4VProcessor(
183 + self.get_hf_config(),
184 + self.get_tokenizer(),
185 + )
186
180 - try:
181 - raw_batch_data = tokenizer.apply_chat_template(
182 - conversation=[{
183 - "role": "user",
184 - "image": multi_modal_data["image"],
185 - "content": inputs['prompt'],
186 - }],
187 - add_generation_prompt=True,
188 - tokenize=True,
189 - return_tensors="pt",
190 - return_dict=True,
191 - ).data
192 - except Exception:
193 - logger.error("Failed to process content (%s)", inputs['prompt'])
194 - raise
195 - input_ids = raw_batch_data['input_ids'][0].tolist()
187
197 - boi_token_id = hf_config.boi_token_id
198 - eoi_token_id = hf_config.eoi_token_id
199 - boi_positions = find_all_positions(input_ids, boi_token_id)
200 - eoi_positions = find_all_positions(input_ids, eoi_token_id)
188 +class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
189
202 - assert len(boi_positions) == len(eoi_positions)
190 + def get_dummy_processor_inputs(
191 + self,
192 + seq_len: int,
193 + mm_counts: Mapping[str, int],
194 + ) -> ProcessorInputs:
195 + num_images = mm_counts.get("image", 0)
196 + target_width, target_height = self.info.get_image_size()
197
204 - new_input_ids = []
205 - final_processed_position = 0
198 + mm_data = {
199 + "image":
200 + self._get_dummy_images(width=target_width,
201 + height=target_height,
202 + num_images=num_images)
203 + }
204 + text = "<|begin_of_image|><|endoftext|><|end_of_image|>"
205 + return ProcessorInputs(
206 + prompt_text=text,
207 + mm_data=mm_data,
208 + )
209
207 - for boi_position, eoi_position in zip(boi_positions, eoi_positions):
208 - assert boi_position < eoi_position
209 - new_input_ids.extend(input_ids[final_processed_position:boi_position +
210 - 1])
211 - new_input_ids.extend([input_ids[boi_position + 1]] *
212 - image_placeholder_length)
213 - final_processed_position = eoi_position
210
215 - new_input_ids.extend(input_ids[final_processed_position:])
211 +class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
212 +
213 + def _get_mm_fields_config(
214 + self,
215 + hf_inputs: BatchFeature,
216 + hf_processor_mm_kwargs: Mapping[str, object],
217 + ) -> Mapping[str, MultiModalFieldConfig]:
218 + return dict(pixel_values=MultiModalFieldConfig.batched("image"))
219 +
220 + def _get_prompt_replacements(
221 + self,
222 + mm_items: MultiModalDataItems,
223 + hf_processor_mm_kwargs: Mapping[str, object],
224 + out_mm_kwargs: MultiModalKwargs,
225 + ) -> list[PromptReplacement]:
226 +
227 + def get_replacement(item_idx: int):
228 + image_tokens = self.info.image_token_num
229 + return [IMAGE_TOKEN_ID] * image_tokens
230 +
231 + return [
232 + PromptReplacement(
233 + modality="image",
234 + target=[IMAGE_TOKEN_ID],
235 + replacement=get_replacement,
236 + ),
237 + ]
238
217 - prompt = inputs.get("prompt")
218 - if prompt is None:
219 - prompt = tokenizer.decode(new_input_ids)
239 + def _apply_prompt_replacements(
240 + self,
241 + token_ids: list[int],
242 + mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
243 + mm_item_counts: Mapping[str, int],
244 + ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
245 + token_ids, text, placeholders = super()._apply_prompt_replacements(
246 + token_ids=token_ids,
247 + mm_prompt_repls=mm_prompt_repls,
248 + mm_item_counts=mm_item_counts,
249 + )
250 + hf_config = self.info.get_hf_config()
251 + boi_token_id = hf_config.boi_token_id
252 + eoi_token_id = hf_config.eoi_token_id
253 + placeholders = {
254 + modality: [
255 + PlaceholderFeaturesInfo(
256 + modality=p.modality,
257 + item_idx=p.item_idx,
258 + start_idx=p.start_idx - 1,
259 + tokens=[boi_token_id] + p.tokens + [eoi_token_id],
260 + ) for p in ps
261 + ]
262 + for modality, ps in placeholders.items()
263 + }
264
221 - return token_inputs(
222 - prompt_token_ids=new_input_ids,
223 - prompt=prompt,
224 - multi_modal_data=multi_modal_data,
225 - )
265 + return token_ids, text, placeholders
266
267
268 class GLMAttention(nn.Module):
@@ -572,12 +612,16 @@ class ChatGLMModel(nn.Module):
612 ) -> torch.Tensor:
613 inputs_embeds = self.embedding(input_ids)
614 if multimodal_embeddings is not None:
575 - inputs_embeds = merge_glm_vision_embeddings(
615 + inputs_embeds = merge_multimodal_embeddings(
616 input_ids=input_ids,
617 inputs_embeds=inputs_embeds,
578 - vision_embeddings=multimodal_embeddings,
579 - boi_token_id=self.config.boi_token_id,
580 - eoi_token_id=self.config.eoi_token_id)
618 + multimodal_embeddings=multimodal_embeddings,
619 + placeholder_token_id=[
620 + self.config.boi_token_id,
621 + IMAGE_TOKEN_ID,
622 + self.config.eoi_token_id,
623 + ],
624 + )
625 return inputs_embeds
626
627 def forward(
@@ -593,14 +637,12 @@ class ChatGLMModel(nn.Module):
637
638 # NOTE: In v1, inputs_embeds is always generated at model runner, this
639 # condition is for v0 compatibility.
596 - if intermediate_tensors is None and inputs_embeds is None:
640 + if intermediate_tensors is not None:
641 + inputs_embeds = intermediate_tensors["hidden_states"]
642 + elif inputs_embeds is None:
643 vision_embeddings = self.get_multimodal_embeddings(**kwargs)
644 inputs_embeds = self.get_input_embeddings(input_ids,
645 vision_embeddings)
600 - input_ids = None
601 - else:
602 - inputs_embeds = intermediate_tensors["hidden_states"]
603 -
646 # Run encoder.
647 hidden_states = self.encoder(
648 hidden_states=inputs_embeds,
@@ -763,11 +805,21 @@ class ChatGLMV(ChatGLMBaseModel, SupportsMultiModal):
805 connector="transformer.vision.linear_proj",
806 tower_model="transformer.vision.transformer")
807
808 + def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
809 + return self.transformer.get_multimodal_embeddings(**kwargs)
810 +
811 + def get_input_embeddings(
812 + self,
813 + input_ids: torch.Tensor,
814 + multimodal_embeddings: Optional[NestedTensors] = None,
815 + ) -> torch.Tensor:
816 + return self.transformer.get_input_embeddings(input_ids,
817 + multimodal_embeddings)
818 +
819
767 -@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
768 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
769 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
770 -@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
820 +@MULTIMODAL_REGISTRY.register_processor(GLM4VMultiModalProcessor,
821 + info=GLM4VProcessingInfo,
822 + dummy_inputs=GLM4VDummyInputsBuilder)
823 class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
824 SupportsMultiModal):
825 # Ensure that the LoRA support check passes when the class is not