@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6cf1167c1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Add GLM-4v support and meet vllm==0.6.2 (#9242)

sixgod committed Oct 12, 2024 at 01:36 UTC 6cf1167c1a82296d1ad6b841138c91698b8f84b0
7 files changed +776 -72
docs/source/models/supported_models.rst
+6
@@ -351,6 +351,12 @@ Text Generation
351 - :code:`adept/fuyu-8b` etc.
352 -
353 - ✅︎
354 + * - :code:`ChatGLMModel`
355 + - GLM-4V
356 + - Image
357 + - :code:`THUDM/glm-4v-9b` etc.
358 + -
359 + - ✅︎
360 * - :code:`InternVLChatModel`
361 - InternVL2
362 - Image\ :sup:`E+`
examples/offline_inference_vision_language.py
+16
@@ -300,6 +300,21 @@ def run_mllama(question: str, modality: str):
300 return llm, prompt, stop_token_ids
301
302
303 +# GLM-4v
304 +def run_glm4v(question: str, modality: str):
305 + assert modality == "image"
306 + model_name = "THUDM/glm-4v-9b"
307 +
308 + llm = LLM(model=model_name,
309 + max_model_len=2048,
310 + max_num_seqs=2,
311 + trust_remote_code=True,
312 + enforce_eager=True)
313 + prompt = question
314 + stop_token_ids = [151329, 151336, 151338]
315 + return llm, prompt, stop_token_ids
316 +
317 +
318 model_example_map = {
319 "llava": run_llava,
320 "llava-next": run_llava_next,
@@ -316,6 +331,7 @@ model_example_map = {
331 "qwen_vl": run_qwen_vl,
332 "qwen2_vl": run_qwen2_vl,
333 "mllama": run_mllama,
334 + "glm4v": run_glm4v,
335 }
336
337
tests/models/decoder_only/vision_language/test_glm4.py new
+133
@@ -0,0 +1,133 @@
1 +from typing import List, Optional, Tuple, Type
2 +
3 +import pytest
4 +
5 +from vllm.multimodal.utils import rescale_image_size
6 +from vllm.transformers_utils.tokenizer import patch_padding_side
7 +
8 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
9 +from ....utils import large_gpu_test
10 +from ...utils import check_logprobs_close
11 +
12 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13 + "stop_sign":
14 + "What's the content of the image?",
15 + "cherry_blossom":
16 + "What is the season?",
17 +})
18 +
19 +models = ["THUDM/glm-4v-9b"]
20 +target_dtype = "bfloat16"
21 +
22 +
23 +def run_test(
24 + hf_runner: Type[HfRunner],
25 + vllm_runner: Type[VllmRunner],
26 + inputs: List[Tuple[List[str], PromptImageInput]],
27 + model: str,
28 + *,
29 + dtype: str,
30 + max_tokens: int,
31 + num_logprobs: int,
32 + mm_limit: int,
33 + tensor_parallel_size: int,
34 + distributed_executor_backend: Optional[str] = None,
35 +):
36 + # max_model_len should be greater than image_feature_size
37 + with vllm_runner(model,
38 + max_model_len=2048,
39 + max_num_seqs=2,
40 + dtype=dtype,
41 + limit_mm_per_prompt={"image": mm_limit},
42 + tensor_parallel_size=tensor_parallel_size,
43 + distributed_executor_backend=distributed_executor_backend,
44 + enforce_eager=True) as vllm_model:
45 + stop_token_ids = [151329, 151336, 151338]
46 + vllm_outputs_per_image = [
47 + vllm_model.generate_greedy_logprobs(prompts,
48 + max_tokens,
49 + num_logprobs=num_logprobs,
50 + images=images,
51 + stop_token_ids=stop_token_ids)
52 + for prompts, images in inputs
53 + ]
54 +
55 + with hf_runner(model, dtype=dtype) as hf_model:
56 + hf_processor = hf_model.processor
57 + patch_padding_side(hf_processor)
58 +
59 + def processor(*args, text="", images=None, **kwargs):
60 + if images is None:
61 + return hf_processor(*args, **kwargs)
62 +
63 + return hf_processor.apply_chat_template(
64 + [{
65 + "role": "user",
66 + "image": images,
67 + "content": text
68 + }],
69 + add_generation_prompt=True,
70 + tokenize=True,
71 + return_dict=True,
72 + **kwargs,
73 + )
74 +
75 + hf_model.processor = processor
76 + hf_model.model.get_output_embeddings = lambda: \
77 + hf_model.model.transformer.output_layer
78 + hf_outputs_per_image = [
79 + hf_model.generate_greedy_logprobs_limit(
80 + prompts,
81 + max_tokens,
82 + num_logprobs=num_logprobs,
83 + images=images,
84 + ) for prompts, images in inputs
85 + ]
86 +
87 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
88 + vllm_outputs_per_image):
89 + check_logprobs_close(
90 + outputs_0_lst=hf_outputs,
91 + outputs_1_lst=vllm_outputs,
92 + name_0="hf",
93 + name_1="vllm",
94 + )
95 +
96 +
97 +@large_gpu_test(min_gb=48)
98 +@pytest.mark.parametrize("model", models)
99 +@pytest.mark.parametrize(
100 + "size_factors",
101 + [
102 + # No image
103 + [],
104 + # Single-scale
105 + [1.0],
106 + # Single-scale, batched
107 + [1.0, 1.0, 1.0],
108 + # Multi-scale
109 + [0.25, 0.5, 1.0],
110 + ],
111 +)
112 +@pytest.mark.parametrize("dtype", [target_dtype])
113 +@pytest.mark.parametrize("max_tokens", [128])
114 +@pytest.mark.parametrize("num_logprobs", [5])
115 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
116 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
117 + images = [asset.pil_image for asset in image_assets]
118 +
119 + inputs_per_image = [(
120 + [prompt for _ in size_factors],
121 + [rescale_image_size(image, factor) for factor in size_factors],
122 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
123 + run_test(
124 + hf_runner,
125 + vllm_runner,
126 + inputs_per_image,
127 + model,
128 + dtype=dtype,
129 + max_tokens=max_tokens,
130 + num_logprobs=num_logprobs,
131 + mm_limit=1,
132 + tensor_parallel_size=1,
133 + )
vllm/model_executor/models/chatglm.py
+298 -52
@@ -1,42 +1,229 @@
1 # coding=utf-8
2 # Adapted from
3 -# https://github.com/THUDM/ChatGLM2-6B
3 +# https://github.com/THUDM/GLM-4
4 """Inference-only ChatGLM model compatible with THUDM weights."""
5 -from typing import Iterable, List, Optional, Tuple, Union
5 +from argparse import Namespace
6 +from array import array
7 +from typing import Dict, Iterable, List, Mapping, Optional, Tuple, TypedDict
8
9 import torch
10 +from PIL import Image
11 from torch import nn
12 from torch.nn import LayerNorm
13
14 from vllm.attention import Attention, AttentionMetadata
12 -from vllm.config import CacheConfig, LoRAConfig
13 -from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
15 +from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16 +from vllm.distributed import get_tensor_model_parallel_world_size
17 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18 +from vllm.logger import init_logger
19 from vllm.model_executor.layers.activation import SiluAndMul
20 from vllm.model_executor.layers.layernorm import RMSNorm
21 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22 QKVParallelLinear,
23 RowParallelLinear)
24 from vllm.model_executor.layers.logits_processor import LogitsProcessor
20 -from vllm.model_executor.layers.quantization import QuantizationConfig
25 +from vllm.model_executor.layers.quantization.base_config import (
26 + QuantizationConfig)
27 from vllm.model_executor.layers.rotary_embedding import get_rope
28 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
29 from vllm.model_executor.layers.vocab_parallel_embedding import (
30 ParallelLMHead, VocabParallelEmbedding)
31 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32 +from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
27 -from vllm.sequence import IntermediateTensors
34 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
35 + MultiModalInputs)
36 +from vllm.multimodal.base import MultiModalData
37 +from vllm.multimodal.utils import cached_get_tokenizer
38 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
39 + SequenceData)
40 from vllm.transformers_utils.configs import ChatGLMConfig
41
30 -from .interfaces import SupportsLoRA, SupportsPP
31 -from .utils import (is_pp_missing_parameter,
32 - make_empty_intermediate_tensors_factory, make_layers)
42 +from .interfaces import SupportsLoRA, SupportsMultiModal
43 +
44 +logger = init_logger(__name__)
45 +
46 +
47 +def calculate_image_placeholder(vision_config):
48 + return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
49 +
50 +
51 +def mm_input_mapper_for_glmv(
52 + ctx: InputContext,
53 + data: MultiModalData[object],
54 +) -> Dict:
55 + model_config = ctx.model_config
56 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
57 + trust_remote_code=True)
58 + if tokenizer is None:
59 + raise RuntimeError("No HuggingFace processor is available "
60 + "to process the image object")
61 + try:
62 + raw_batch_data = tokenizer.apply_chat_template(
63 + conversation=[{
64 + "role": "user",
65 + "image": data
66 + }],
67 + add_generation_prompt=True,
68 + tokenize=True,
69 + return_tensors="pt",
70 + return_dict=True).data
71 + except Exception:
72 + logger.error("Failed to process image (%s)", data)
73 + raise
74 + pixel_values = raw_batch_data['images']
75 +
76 + return MultiModalInputs({'pixel_values': pixel_values})
77 +
78 +
79 +def merge_glm_vision_embeddings(
80 + input_ids: torch.Tensor,
81 + inputs_embeds: torch.Tensor,
82 + vision_embeddings: torch.Tensor,
83 + boi_token_id: int,
84 + eoi_token_id: int,
85 +) -> torch.Tensor:
86 +
87 + boi_positions = (input_ids == boi_token_id).nonzero(as_tuple=True)[0]
88 + eoi_positions = (input_ids == eoi_token_id).nonzero(as_tuple=True)[0]
89 +
90 + mask = torch.zeros_like(input_ids, dtype=torch.bool)
91 +
92 + for boi_pos, eoi_pos in zip(boi_positions, eoi_positions):
93 + assert boi_pos < eoi_pos
94 + mask[boi_pos:eoi_pos + 1] = True
95 + inputs_embeds[mask] = vision_embeddings.view(-1,
96 + vision_embeddings.shape[-1])
97 + return inputs_embeds
98 +
99 +
100 +class GLMImagePixelInputs(TypedDict):
101 + pixel_values: torch.Tensor
102 + """Shape: `(batch_size, num_channels, height, width)`"""
103 +
104 +
105 +def get_max_glmv_image_tokens(ctx: InputContext):
106 + hf_config = ctx.get_hf_config(ChatGLMConfig)
107 +
108 + vision_config = getattr(hf_config, 'vision_config', None)
109 + if vision_config is None:
110 + return 1
111 + elif isinstance(vision_config, dict):
112 + return calculate_image_placeholder(vision_config)
113 +
114 + msg = f"Unsupported vision config: {type(vision_config)}"
115 + raise NotImplementedError(msg)
116 +
117 +
118 +def dummy_data_for_glmv(
119 + ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
120 +) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
121 + hf_config = ctx.get_hf_config(ChatGLMConfig)
122 + vision_config = getattr(hf_config, 'vision_config', None)
123 +
124 + if vision_config is None:
125 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len)
126 + seq_data = SequenceData(token_ids)
127 + return seq_data, None
128 + elif isinstance(vision_config, dict):
129 + image_size = vision_config["image_size"]
130 + image_placeholder_length = calculate_image_placeholder(vision_config)
131 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [hf_config.boi_token_id] +
132 + [0] * image_placeholder_length +
133 + [hf_config.eoi_token_id])
134 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
135 + [0] * (seq_len - image_placeholder_length - 2))
136 + seq_data = SequenceData(token_ids)
137 +
138 + mm_data = {
139 + "image": Image.new("RGB", (image_size, image_size), color=0)
140 + }
141 +
142 + return seq_data, mm_data
143 +
144 + msg = f"Unsupported vision config: {type(vision_config)}"
145 + raise NotImplementedError(msg)
146 +
147 +
148 +def find_all_positions(input_ids: List[int], target: int) -> List[int]:
149 + return [index for index, value in enumerate(input_ids) if value == target]
150 +
151 +
152 +def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
153 + hf_config = ctx.get_hf_config(ChatGLMConfig)
154 + vision_config = getattr(hf_config, 'vision_config', None)
155 +
156 + if vision_config is None:
157 + return llm_inputs
158 + elif isinstance(vision_config, dict):
159 + image_placeholder_length = calculate_image_placeholder(vision_config)
160 + else:
161 + msg = f"Unsupported vision config: {type(vision_config)}"
162 + raise NotImplementedError(msg)
163 +
164 + input_ids = llm_inputs.get("prompt_token_ids")
165 + position_ids = llm_inputs.get("position_ids")
166 + tokenizer = cached_get_tokenizer(
167 + ctx.model_config.model,
168 + trust_remote_code=ctx.model_config.trust_remote_code)
169 +
170 + try:
171 + raw_batch_data = tokenizer.apply_chat_template(
172 + conversation=[{
173 + "role": "user",
174 + "image": llm_inputs['multi_modal_data']["image"],
175 + "content": llm_inputs['prompt']
176 + }],
177 + add_generation_prompt=True,
178 + tokenize=True,
179 + return_tensors="pt",
180 + return_dict=True).data
181 + except Exception:
182 + logger.error("Failed to process content (%s)", llm_inputs['prompt'])
183 + raise
184 + input_ids = raw_batch_data['input_ids'][0].tolist()
185 +
186 + if position_ids is None:
187 + position_ids = list(range(len(input_ids)))
188 + boi_token_id = hf_config.boi_token_id
189 + eoi_token_id = hf_config.eoi_token_id
190 + boi_positions = find_all_positions(input_ids, boi_token_id)
191 + eoi_positions = find_all_positions(input_ids, eoi_token_id)
192 +
193 + assert len(boi_positions) == len(eoi_positions)
194 +
195 + new_input_ids = []
196 + new_position_ids = []
197 + final_processed_position = 0
198 + final_processed_position = 0
199 +
200 + for boi_position, eoi_position in zip(boi_positions, eoi_positions):
201 + assert boi_position < eoi_position
202 + new_input_ids.extend(input_ids[final_processed_position:boi_position +
203 + 1])
204 + new_position_ids.extend(
205 + list(range(final_processed_position, boi_position + 1)))
206 + new_input_ids.extend([input_ids[boi_position + 1]] *
207 + image_placeholder_length)
208 + new_position_ids.extend([boi_position + 1] * image_placeholder_length)
209 + final_processed_position = eoi_position
210 +
211 + new_input_ids.extend(input_ids[final_processed_position:])
212 + new_position_ids.extend(
213 + list(range(final_processed_position, len(input_ids))))
214 +
215 + assert len(new_input_ids) == len(new_position_ids)
216 +
217 + llm_inputs["prompt_token_ids"] = new_input_ids
218 + llm_inputs["position_ids"] = new_position_ids
219 + return llm_inputs
220
221
222 class GLMAttention(nn.Module):
223
224 def __init__(
225 self,
39 - config: ChatGLMConfig,
226 + config,
227 cache_config: Optional[CacheConfig] = None,
228 quant_config: Optional[QuantizationConfig] = None,
229 ):
@@ -127,7 +314,7 @@ class GLMMLP(nn.Module):
314
315 def __init__(
316 self,
130 - config: ChatGLMConfig,
317 + config,
318 quant_config: Optional[QuantizationConfig] = None,
319 ):
320 super().__init__()
@@ -170,7 +357,7 @@ class GLMBlock(nn.Module):
357
358 def __init__(
359 self,
173 - config: ChatGLMConfig,
360 + config,
361 cache_config: Optional[CacheConfig] = None,
362 quant_config: Optional[QuantizationConfig] = None,
363 ):
@@ -241,10 +428,9 @@ class GLMTransformer(nn.Module):
428
429 def __init__(
430 self,
244 - config: ChatGLMConfig,
431 + config,
432 cache_config: Optional[CacheConfig] = None,
433 quant_config: Optional[QuantizationConfig] = None,
247 - prefix: str = "",
434 ):
435 super().__init__()
436 self.post_layer_norm = config.post_layer_norm
@@ -253,11 +439,10 @@ class GLMTransformer(nn.Module):
439 self.num_layers = config.num_layers
440
441 # Transformer layers.
256 - self.start_layer, self.end_layer, self.layers = make_layers(
257 - self.num_layers,
258 - lambda prefix: GLMBlock(config, cache_config, quant_config),
259 - prefix=f"{prefix}.layers",
260 - )
442 + self.layers = nn.ModuleList([
443 + GLMBlock(config, cache_config, quant_config)
444 + for i in range(self.num_layers)
445 + ])
446
447 if self.post_layer_norm:
448 layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -272,16 +457,16 @@ class GLMTransformer(nn.Module):
457 kv_caches: List[torch.Tensor],
458 attn_metadata: AttentionMetadata,
459 ) -> torch.Tensor:
275 - for i in range(self.start_layer, self.end_layer):
460 + for i in range(self.num_layers):
461 layer = self.layers[i]
462 hidden_states = layer(
463 hidden_states=hidden_states,
464 position_ids=position_ids,
280 - kv_cache=kv_caches[i - self.start_layer],
465 + kv_cache=kv_caches[i],
466 attn_metadata=attn_metadata,
467 )
468 # Final layer norm.
284 - if get_pp_group().is_last_rank and self.post_layer_norm:
469 + if self.post_layer_norm:
470 hidden_states = self.final_layernorm(hidden_states)
471
472 return hidden_states
@@ -291,14 +476,17 @@ class ChatGLMModel(nn.Module):
476
477 def __init__(
478 self,
294 - config: ChatGLMConfig,
479 + config,
480 cache_config: Optional[CacheConfig] = None,
481 quant_config: Optional[QuantizationConfig] = None,
482 ):
483 super().__init__()
484
485 + self.config = config
486 +
487 self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
301 - config.hidden_size)
488 + config.hidden_size,
489 + quant_config=quant_config)
490
491 self.num_layers = config.num_layers
492 self.multi_query_group_num = config.multi_query_group_num
@@ -308,37 +496,73 @@ class ChatGLMModel(nn.Module):
496 self.output_layer = ParallelLMHead(config.padded_vocab_size,
497 config.hidden_size,
498 quant_config=quant_config)
311 - self.make_empty_intermediate_tensors = (
312 - make_empty_intermediate_tensors_factory(["hidden_states"],
313 - config.hidden_size))
499 +
500 + vision_config_flag = getattr(config, 'vision_config', None)
501 + if vision_config_flag is not None:
502 + self.vision_config = Namespace(**config.vision_config)
503 + self.vision = EVA2CLIPModel(self.config, quant_config)
504 + else:
505 + self.vision = None
506 +
507 + def _parse_and_validate_image_input(
508 + self, **kwargs: object) -> GLMImagePixelInputs:
509 +
510 + pixel_values = kwargs.pop("pixel_values", None)
511 + if pixel_values is not None and self.vision is not None:
512 + if isinstance(pixel_values, torch.Tensor):
513 + if pixel_values.ndim > 2:
514 + pixel_values = torch.concat(list(pixel_values))
515 + elif isinstance(pixel_values, list):
516 + return torch.concat(pixel_values)
517 + else:
518 + raise TypeError("""pixel_values must be a torch.Tensor
519 + or a list of torch.Tensor
520 + """)
521 + return GLMImagePixelInputs(pixel_values=pixel_values)
522
523 def forward(
524 self,
525 input_ids: torch.Tensor,
318 - position_ids: torch.Tensor,
526 + positions: torch.Tensor,
527 kv_caches: List[torch.Tensor],
528 attn_metadata: AttentionMetadata,
321 - intermediate_tensors: Optional[IntermediateTensors],
322 - ) -> Union[torch.Tensor, IntermediateTensors]:
323 - if get_pp_group().is_first_rank:
324 - inputs_embeds = self.embedding(input_ids)
325 - else:
326 - inputs_embeds = intermediate_tensors["hidden_states"]
529 + intermediate_tensors: Optional[IntermediateTensors] = None,
530 + **kwargs: object,
531 + ) -> torch.Tensor:
532 +
533 + inputs_embeds = self.embedding(input_ids)
534 + image_input = self._parse_and_validate_image_input(**kwargs)
535 +
536 + if image_input["pixel_values"] is not None:
537 + pixel_values = image_input["pixel_values"].to(
538 + dtype=inputs_embeds.dtype)
539 + image_embeds = self.vision(pixel_values)
540 +
541 + boi_token_id = self.config.boi_token_id
542 + eoi_token_id = self.config.eoi_token_id
543 +
544 + inputs_embeds = merge_glm_vision_embeddings(
545 + input_ids=input_ids,
546 + inputs_embeds=inputs_embeds,
547 + vision_embeddings=image_embeds,
548 + boi_token_id=boi_token_id,
549 + eoi_token_id=eoi_token_id)
550
551 # Run encoder.
552 hidden_states = self.encoder(
553 hidden_states=inputs_embeds,
331 - position_ids=position_ids,
554 + position_ids=positions,
555 kv_caches=kv_caches,
556 attn_metadata=attn_metadata,
557 )
335 -
336 - if not get_pp_group().is_last_rank:
337 - return IntermediateTensors({"hidden_states": hidden_states})
558 return hidden_states
559
560
341 -class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
561 +@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
562 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
563 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
564 +@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
565 +class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
566 packed_modules_mapping = {
567 "query_key_value": ["query_key_value"],
568 "dense_h_to_4h": ["dense_h_to_4h"]
@@ -356,6 +580,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
580 def __init__(
581 self,
582 config: ChatGLMConfig,
583 + multimodal_config: MultiModalConfig,
584 cache_config: Optional[CacheConfig] = None,
585 quant_config: Optional[QuantizationConfig] = None,
586 lora_config: Optional[LoRAConfig] = None,
@@ -364,6 +589,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
589
590 self.config = config
591 self.lora_config = lora_config
592 + self.multimodal_config = multimodal_config
593
594 self.quant_config = quant_config
595 self.max_position_embeddings = getattr(config, "max_sequence_length",
@@ -375,19 +601,16 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
601 self.lm_head = self.transformer.output_layer
602 self.logits_processor = LogitsProcessor(config.padded_vocab_size)
603 self.sampler = Sampler()
378 - self.make_empty_intermediate_tensors = (
379 - self.transformer.make_empty_intermediate_tensors)
604
381 - def forward(
382 - self,
383 - input_ids: torch.Tensor,
384 - positions: torch.Tensor,
385 - kv_caches: List[torch.Tensor],
386 - attn_metadata: AttentionMetadata,
387 - intermediate_tensors: Optional[IntermediateTensors] = None,
388 - ) -> Union[torch.Tensor, IntermediateTensors]:
605 + def forward(self,
606 + input_ids: torch.Tensor,
607 + positions: torch.Tensor,
608 + kv_caches: List[torch.Tensor],
609 + attn_metadata: AttentionMetadata,
610 + intermediate_tensors: Optional[IntermediateTensors] = None,
611 + **kwargs) -> torch.Tensor:
612 hidden_states = self.transformer(input_ids, positions, kv_caches,
390 - attn_metadata, intermediate_tensors)
613 + attn_metadata, **kwargs)
614 return hidden_states
615
616 def compute_logits(
@@ -408,8 +631,24 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
631 return next_tokens
632
633 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
634 + # Merge two ColumnParallelLinear into one MergedColumnParallelLinear
635 + merged_weights_dict: Dict[str, Dict[str, Optional[torch.Tensor]]] = {
636 + "transformer.vision.linear_proj.merged_proj.weight": {
637 + "transformer.vision.linear_proj.gate_proj.weight": None,
638 + "transformer.vision.linear_proj.dense_h_to_4h.weight": None,
639 + }
640 + }
641 +
642 params_dict = dict(self.named_parameters(remove_duplicate=False))
643 for name, loaded_weight in weights:
644 + is_weight_to_be_merge = False
645 + for _, merged_weight_dict in merged_weights_dict.items():
646 + if name in merged_weight_dict:
647 + assert merged_weight_dict[name] is None
648 + merged_weight_dict[name] = loaded_weight
649 + is_weight_to_be_merge = True
650 + if is_weight_to_be_merge:
651 + continue
652 if "rotary_pos_emb.inv_freq" in name:
653 continue
654 if "word_embeddings" in name:
@@ -417,9 +656,16 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
656 # Skip loading extra bias for GPTQ models.
657 if name.endswith(".bias") and name not in params_dict:
658 continue
420 - if is_pp_missing_parameter(name, self):
421 - continue
659 param = params_dict[name]
660 weight_loader = getattr(param, "weight_loader",
661 default_weight_loader)
662 weight_loader(param, loaded_weight)
663 +
664 + for combined_name, merged_weight_dict in merged_weights_dict.items():
665 + if combined_name in params_dict:
666 + param = params_dict[combined_name]
667 + combined_weight = torch.cat(list(merged_weight_dict.values()),
668 + dim=0)
669 + weight_loader = getattr(param, "weight_loader",
670 + default_weight_loader)
671 + weight_loader(param, combined_weight)
vllm/model_executor/models/glm4_vision_encoder.py new
+298
@@ -0,0 +1,298 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/THUDM/GLM-4
4 +"""Inference-only GLM-4v model visual encoder compatible with THUDM weights."""
5 +from argparse import Namespace
6 +from typing import Optional
7 +
8 +import torch
9 +from torch import nn
10 +from torch.nn import LayerNorm
11 +
12 +from vllm.distributed import get_tensor_model_parallel_world_size
13 +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
14 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
15 + MergedColumnParallelLinear,
16 + QKVParallelLinear,
17 + ReplicatedLinear,
18 + RowParallelLinear)
19 +from vllm.model_executor.layers.quantization.base_config import (
20 + QuantizationConfig)
21 +
22 +
23 +class PatchEmbedding(nn.Module):
24 +
25 + def __init__(self, config):
26 + super().__init__()
27 + self.proj = nn.Conv2d(config.in_channels,
28 + config.hidden_size,
29 + kernel_size=config.patch_size,
30 + stride=config.patch_size)
31 + self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size))
32 + self.position_embedding = nn.Embedding(config.num_positions,
33 + config.hidden_size)
34 +
35 + def forward(self, images: torch.Tensor) -> torch.Tensor:
36 + """
37 + Parameters:
38 + images : torch.Tensor
39 + Input image tensor with shape (B, C, H, W)
40 +
41 + Returns:
42 + torch.Tensor
43 + Transformed tensor with shape (B, L, D)
44 + """
45 + images = images.to(self.proj.weight.device)
46 + x = self.proj(images)
47 + x = x.flatten(2).transpose(1, 2)
48 + cls_token = self.cls_embedding.expand(x.shape[0], -1, -1)
49 + x = torch.cat((cls_token, x), dim=1)
50 + x += self.position_embedding.weight.unsqueeze(0)
51 + return x
52 +
53 +
54 +class Attention(nn.Module):
55 +
56 + def __init__(
57 + self,
58 + config,
59 + quant_config: Optional[QuantizationConfig] = None,
60 + ):
61 + super().__init__()
62 + self.hidden_size = config.hidden_size
63 + self.tp_size = get_tensor_model_parallel_world_size()
64 + self.num_heads_per_rank = config.num_heads // self.tp_size
65 + self.head_dim = config.hidden_size // config.num_heads
66 + self.scale = self.head_dim**-0.5
67 +
68 + self.query_key_value = QKVParallelLinear(
69 + config.hidden_size,
70 + self.head_dim,
71 + config.num_heads,
72 + quant_config=quant_config,
73 + )
74 + self.dense = RowParallelLinear(
75 + config.hidden_size,
76 + config.hidden_size,
77 + quant_config=quant_config,
78 + )
79 +
80 + self.output_dropout = torch.nn.Dropout(config.dropout_prob)
81 +
82 + def forward(self, x: torch.Tensor) -> torch.Tensor:
83 + B, L, _ = x.shape
84 + qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
85 + q, k, v = qkv.chunk(3, dim=-1)
86 + q = q.reshape(B, L, self.num_heads_per_rank,
87 + self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
88 + k = k.reshape(B, L, self.num_heads_per_rank,
89 + self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
90 + v = v.reshape(B, L, self.num_heads_per_rank,
91 + self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
92 +
93 + out = torch.nn.functional.scaled_dot_product_attention(q,
94 + k,
95 + v,
96 + attn_mask=None,
97 + dropout_p=0.,
98 + is_causal=False)
99 +
100 + output, _ = self.dense(out.transpose(1, 2).view(B, L, -1))
101 + output = self.output_dropout(output)
102 + return output
103 +
104 +
105 +class MLP(nn.Module):
106 +
107 + def __init__(
108 + self,
109 + config,
110 + quant_config: Optional[QuantizationConfig] = None,
111 + ):
112 + super().__init__()
113 + self.config = config
114 + self.activation_fn = get_act_fn(config.hidden_act)
115 + self.fc1 = ColumnParallelLinear(
116 + config.hidden_size,
117 + config.intermediate_size,
118 + quant_config=quant_config,
119 + )
120 + self.fc2 = RowParallelLinear(
121 + config.intermediate_size,
122 + config.hidden_size,
123 + quant_config=quant_config,
124 + )
125 +
126 + def forward(self, x: torch.Tensor) -> torch.Tensor:
127 + x, _ = self.fc1(x)
128 + x = self.activation_fn(x)
129 + x, _ = self.fc2(x)
130 + return x
131 +
132 +
133 +class TransformerLayer(nn.Module):
134 +
135 + def __init__(
136 + self,
137 + config,
138 + quant_config: Optional[QuantizationConfig] = None,
139 + ):
140 + super().__init__()
141 + self.input_layernorm = LayerNorm(config.hidden_size,
142 + eps=config.layer_norm_eps)
143 + self.attention = Attention(config, quant_config=quant_config)
144 + self.mlp = MLP(config, quant_config=quant_config)
145 + self.post_attention_layernorm = LayerNorm(config.hidden_size,
146 + eps=config.layer_norm_eps)
147 +
148 + def forward(self, hidden_states):
149 + attention_input = hidden_states
150 + attention_output = self.input_layernorm(
151 + self.attention(attention_input))
152 + hidden_states = attention_input + attention_output
153 + mlp_input = hidden_states
154 + mlp_output = self.post_attention_layernorm(self.mlp(mlp_input))
155 + output = mlp_input + mlp_output
156 + return output
157 +
158 +
159 +class Transformer(nn.Module):
160 +
161 + def __init__(
162 + self,
163 + config,
164 + quant_config: Optional[QuantizationConfig] = None,
165 + ):
166 + super().__init__()
167 + self.layers = nn.ModuleList([
168 + TransformerLayer(config, quant_config=quant_config)
169 + for _ in range(config.num_hidden_layers)
170 + ])
171 +
172 + def forward(self, hidden_states):
173 + for layer_module in self.layers:
174 + hidden_states = layer_module(hidden_states)
175 + return hidden_states
176 +
177 +
178 +class GLU(nn.Module):
179 +
180 + def __init__(
181 + self,
182 + config,
183 + in_features,
184 + quant_config: Optional[QuantizationConfig] = None,
185 + ):
186 + """
187 + The original implementation is the same as:
188 + ```python
189 + self.dense_h_to_4h = ColumnParallelLinear(
190 + config.hidden_size,
191 + config.ffn_hidden_size,
192 + bias=False,
193 + quant_config=quant_config
194 + )
195 +
196 + self.gate_proj = ColumnParallelLinear(
197 + config.hidden_size,
198 + config.ffn_hidden_size,
199 + bias=False,
200 + quant_config=quant_config
201 + )
202 + ```
203 + ```
204 + gate_proj_output, _ = self.gate_proj(x)
205 + dense_h_to_4h_output, _ = self.dense_h_to_4h(x)
206 + x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1)
207 + ```
208 +
209 + We merge two ColumnParallelLinear into one MergedColumnParallelLinear:
210 + ```
211 + self.merged_proj = MergedColumnParallelLinear(
212 + config.hidden_size,
213 + [config.ffn_hidden_size] * 2,
214 + bias=False,
215 + quant_config=quant_config
216 + )
217 + ```
218 + ```
219 + x, _ = self.merged_proj(x)
220 + ```
221 + """
222 + super().__init__()
223 + self.linear_proj = ReplicatedLinear(in_features,
224 + config.hidden_size,
225 + bias=False,
226 + quant_config=quant_config)
227 + self.norm1 = nn.LayerNorm(config.hidden_size)
228 + self.act1 = nn.GELU()
229 + self.act2 = SiluAndMul()
230 +
231 + self.merged_proj = MergedColumnParallelLinear(
232 + config.hidden_size, [config.ffn_hidden_size] * 2,
233 + bias=False,
234 + quant_config=quant_config)
235 +
236 + self.dense_4h_to_h = RowParallelLinear(config.ffn_hidden_size,
237 + config.hidden_size,
238 + bias=False,
239 + quant_config=quant_config)
240 +
241 + def forward(self, x):
242 + x, _ = self.linear_proj(x)
243 + x = self.act1(self.norm1(x))
244 + x, _ = self.merged_proj(x)
245 + x = self.act2(x)
246 + x, _ = self.dense_4h_to_h(x)
247 + return x
248 +
249 +
250 +class EVA2CLIPModel(nn.Module):
251 +
252 + def __init__(
253 + self,
254 + config,
255 + quant_config: Optional[QuantizationConfig] = None,
256 + ):
257 + super().__init__()
258 + vision_config = Namespace(**config.vision_config)
259 + self.patch_embedding = PatchEmbedding(vision_config)
260 + self.transformer = Transformer(vision_config,
261 + quant_config=quant_config)
262 + self.linear_proj = GLU(config,
263 + in_features=config.hidden_size,
264 + quant_config=quant_config)
265 + self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
266 + out_channels=config.hidden_size,
267 + kernel_size=2,
268 + stride=2)
269 + self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
270 + self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
271 + self.scaling_factor = vision_config.scaling_factor
272 +
273 + def forward(self, images: torch.Tensor) -> torch.Tensor:
274 + """
275 + Parameters:
276 + images : torch.Tensor
277 + Input image tensor with shape (B, C, H, W)
278 +
279 + Returns:
280 + torch.Tensor
281 + Transformed tensor with shape (B, L, D)
282 + """
283 + x = self.patch_embedding(images)
284 + x = self.transformer(x)
285 + x = x[:, 1:]
286 +
287 + b, s, h = x.shape
288 + grid_size = int(s**0.5)
289 + x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2)
290 + x = self.conv(x)
291 +
292 + x = x.flatten(2).transpose(1, 2)
293 + x = self.linear_proj(x)
294 + boi = self.boi.expand(x.shape[0], -1, -1)
295 + eoi = self.eoi.expand(x.shape[0], -1, -1)
296 + x = torch.cat((boi, x, eoi), dim=1)
297 + x = x / self.scaling_factor
298 + return x
vllm/model_executor/models/registry.py
+4 -2
@@ -29,8 +29,7 @@ _TEXT_GENERATION_MODELS = {
29 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
30 "BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
31 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
32 - "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
33 - "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
32 + # ChatGLMModel supports multimodal
33 "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
34 "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
35 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
@@ -72,6 +71,7 @@ _TEXT_GENERATION_MODELS = {
71 "Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
72 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
73 "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
74 + # QWenLMHeadModel supports multimodal
75 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
76 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
77 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
@@ -95,6 +95,8 @@ _MULTIMODAL_MODELS = {
95 # [Decoder-only]
96 "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
97 "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
98 + "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
99 + "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
100 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
101 "InternVLChatModel": ("internvl", "InternVLChatModel"),
102 "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
vllm/transformers_utils/tokenizer.py
+21 -18
@@ -59,6 +59,26 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
59 return tokenizer
60
61
62 +def patch_padding_side(tokenizer: PreTrainedTokenizer) -> None:
63 + """Patch _pad method to accept `padding_side` for older tokenizers."""
64 + orig_pad = tokenizer._pad
65 +
66 + def _pad(
67 + self: PreTrainedTokenizer,
68 + *args,
69 + padding_side: Optional[str] = None,
70 + **kwargs,
71 + ):
72 + if padding_side is not None and padding_side != self.padding_side:
73 + msg = ("`padding_side` argument is not supported by "
74 + f"{type(tokenizer).__name__} and will be ignored.")
75 + warnings.warn(msg, stacklevel=2)
76 +
77 + return orig_pad(*args, **kwargs)
78 +
79 + tokenizer._pad = MethodType(_pad, tokenizer)
80 +
81 +
82 def get_tokenizer(
83 tokenizer_name: Union[str, Path],
84 *args,
@@ -143,24 +163,7 @@ def get_tokenizer(
163 if type(tokenizer).__name__ in ("ChatGLMTokenizer",
164 "ChatGLM4Tokenizer"):
165 assert isinstance(tokenizer, PreTrainedTokenizer)
146 - orig_pad = tokenizer._pad
147 -
148 - # Patch _pad method to accept `padding_side`
149 - def _pad(
150 - self: PreTrainedTokenizer,
151 - *args,
152 - padding_side: Optional[str] = None,
153 - **kwargs,
154 - ):
155 - if (padding_side is not None
156 - and padding_side != self.padding_side):
157 - msg = ("`padding_side` argument is not supported by "
158 - "ChatGLMTokenizer and will be ignored.")
159 - warnings.warn(msg, stacklevel=2)
160 -
161 - return orig_pad(*args, **kwargs)
162 -
163 - tokenizer._pad = MethodType(_pad, tokenizer)
166 + patch_padding_side(tokenizer)
167
168 if not isinstance(tokenizer, PreTrainedTokenizerFast):
169 logger.warning(