A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Add GLM-4v support and meet vllm==0.6.2 (#9242)
sixgod committed
Oct 12, 2024 at 01:36 UTC
6cf1167c1a82296d1ad6b841138c91698b8f84b0
7 files changed
+776
-72
docs/source/models/supported_models.rst
+6
@@ -351,6 +351,12 @@ Text Generation
351
- :code:`adept/fuyu-8b` etc.
352
-
353
- ✅︎
354
+ * - :code:`ChatGLMModel`
355
+ - GLM-4V
356
+ - Image
357
+ - :code:`THUDM/glm-4v-9b` etc.
358
+ -
359
+ - ✅︎
360
* - :code:`InternVLChatModel`
361
- InternVL2
362
- Image\ :sup:`E+`
examples/offline_inference_vision_language.py
+16
@@ -300,6 +300,21 @@ def run_mllama(question: str, modality: str):
300
return llm, prompt, stop_token_ids
301
302
303
+# GLM-4v
304
+def run_glm4v(question: str, modality: str):
305
+ assert modality == "image"
306
+ model_name = "THUDM/glm-4v-9b"
307
+
308
+ llm = LLM(model=model_name,
309
+ max_model_len=2048,
310
+ max_num_seqs=2,
311
+ trust_remote_code=True,
312
+ enforce_eager=True)
313
+ prompt = question
314
+ stop_token_ids = [151329, 151336, 151338]
315
+ return llm, prompt, stop_token_ids
316
+
317
+
318
model_example_map = {
319
"llava": run_llava,
320
"llava-next": run_llava_next,
@@ -316,6 +331,7 @@ model_example_map = {
331
"qwen_vl": run_qwen_vl,
332
"qwen2_vl": run_qwen2_vl,
333
"mllama": run_mllama,
334
+ "glm4v": run_glm4v,
335
}
336
337
tests/models/decoder_only/vision_language/test_glm4.py
new
+133
@@ -0,0 +1,133 @@
1
+from typing import List, Optional, Tuple, Type
2
+
3
+import pytest
4
+
5
+from vllm.multimodal.utils import rescale_image_size
6
+from vllm.transformers_utils.tokenizer import patch_padding_side
7
+
8
+from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
9
+from ....utils import large_gpu_test
10
+from ...utils import check_logprobs_close
11
+
12
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13
+ "stop_sign":
14
+ "What's the content of the image?",
15
+ "cherry_blossom":
16
+ "What is the season?",
17
+})
18
+
19
+models = ["THUDM/glm-4v-9b"]
20
+target_dtype = "bfloat16"
21
+
22
+
23
+def run_test(
24
+ hf_runner: Type[HfRunner],
25
+ vllm_runner: Type[VllmRunner],
26
+ inputs: List[Tuple[List[str], PromptImageInput]],
27
+ model: str,
28
+ *,
29
+ dtype: str,
30
+ max_tokens: int,
31
+ num_logprobs: int,
32
+ mm_limit: int,
33
+ tensor_parallel_size: int,
34
+ distributed_executor_backend: Optional[str] = None,
35
+):
36
+ # max_model_len should be greater than image_feature_size
37
+ with vllm_runner(model,
38
+ max_model_len=2048,
39
+ max_num_seqs=2,
40
+ dtype=dtype,
41
+ limit_mm_per_prompt={"image": mm_limit},
42
+ tensor_parallel_size=tensor_parallel_size,
43
+ distributed_executor_backend=distributed_executor_backend,
44
+ enforce_eager=True) as vllm_model:
45
+ stop_token_ids = [151329, 151336, 151338]
46
+ vllm_outputs_per_image = [
47
+ vllm_model.generate_greedy_logprobs(prompts,
48
+ max_tokens,
49
+ num_logprobs=num_logprobs,
50
+ images=images,
51
+ stop_token_ids=stop_token_ids)
52
+ for prompts, images in inputs
53
+ ]
54
+
55
+ with hf_runner(model, dtype=dtype) as hf_model:
56
+ hf_processor = hf_model.processor
57
+ patch_padding_side(hf_processor)
58
+
59
+ def processor(*args, text="", images=None, **kwargs):
60
+ if images is None:
61
+ return hf_processor(*args, **kwargs)
62
+
63
+ return hf_processor.apply_chat_template(
64
+ [{
65
+ "role": "user",
66
+ "image": images,
67
+ "content": text
68
+ }],
69
+ add_generation_prompt=True,
70
+ tokenize=True,
71
+ return_dict=True,
72
+ **kwargs,
73
+ )
74
+
75
+ hf_model.processor = processor
76
+ hf_model.model.get_output_embeddings = lambda: \
77
+ hf_model.model.transformer.output_layer
78
+ hf_outputs_per_image = [
79
+ hf_model.generate_greedy_logprobs_limit(
80
+ prompts,
81
+ max_tokens,
82
+ num_logprobs=num_logprobs,
83
+ images=images,
84
+ ) for prompts, images in inputs
85
+ ]
86
+
87
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
88
+ vllm_outputs_per_image):
89
+ check_logprobs_close(
90
+ outputs_0_lst=hf_outputs,
91
+ outputs_1_lst=vllm_outputs,
92
+ name_0="hf",
93
+ name_1="vllm",
94
+ )
95
+
96
+
97
+@large_gpu_test(min_gb=48)
98
+@pytest.mark.parametrize("model", models)
99
+@pytest.mark.parametrize(
100
+ "size_factors",
101
+ [
102
+ # No image
103
+ [],
104
+ # Single-scale
105
+ [1.0],
106
+ # Single-scale, batched
107
+ [1.0, 1.0, 1.0],
108
+ # Multi-scale
109
+ [0.25, 0.5, 1.0],
110
+ ],
111
+)
112
+@pytest.mark.parametrize("dtype", [target_dtype])
113
+@pytest.mark.parametrize("max_tokens", [128])
114
+@pytest.mark.parametrize("num_logprobs", [5])
115
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
116
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
117
+ images = [asset.pil_image for asset in image_assets]
118
+
119
+ inputs_per_image = [(
120
+ [prompt for _ in size_factors],
121
+ [rescale_image_size(image, factor) for factor in size_factors],
122
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
123
+ run_test(
124
+ hf_runner,
125
+ vllm_runner,
126
+ inputs_per_image,
127
+ model,
128
+ dtype=dtype,
129
+ max_tokens=max_tokens,
130
+ num_logprobs=num_logprobs,
131
+ mm_limit=1,
132
+ tensor_parallel_size=1,
133
+ )
vllm/model_executor/models/chatglm.py
+298
-52
@@ -1,42 +1,229 @@
1
# coding=utf-8
2
# Adapted from
3
-# https://github.com/THUDM/ChatGLM2-6B
3
+# https://github.com/THUDM/GLM-4
4
"""Inference-only ChatGLM model compatible with THUDM weights."""
5
-from typing import Iterable, List, Optional, Tuple, Union
5
+from argparse import Namespace
6
+from array import array
7
+from typing import Dict, Iterable, List, Mapping, Optional, Tuple, TypedDict
8
9
import torch
10
+from PIL import Image
11
from torch import nn
12
from torch.nn import LayerNorm
13
14
from vllm.attention import Attention, AttentionMetadata
12
-from vllm.config import CacheConfig, LoRAConfig
13
-from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
15
+from vllm.config import CacheConfig, LoRAConfig, MultiModalConfig
16
+from vllm.distributed import get_tensor_model_parallel_world_size
17
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
18
+from vllm.logger import init_logger
19
from vllm.model_executor.layers.activation import SiluAndMul
20
from vllm.model_executor.layers.layernorm import RMSNorm
21
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22
QKVParallelLinear,
23
RowParallelLinear)
24
from vllm.model_executor.layers.logits_processor import LogitsProcessor
20
-from vllm.model_executor.layers.quantization import QuantizationConfig
25
+from vllm.model_executor.layers.quantization.base_config import (
26
+ QuantizationConfig)
27
from vllm.model_executor.layers.rotary_embedding import get_rope
28
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
29
from vllm.model_executor.layers.vocab_parallel_embedding import (
30
ParallelLMHead, VocabParallelEmbedding)
31
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32
+from vllm.model_executor.models.glm4_vision_encoder import EVA2CLIPModel
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
27
-from vllm.sequence import IntermediateTensors
34
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
35
+ MultiModalInputs)
36
+from vllm.multimodal.base import MultiModalData
37
+from vllm.multimodal.utils import cached_get_tokenizer
38
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
39
+ SequenceData)
40
from vllm.transformers_utils.configs import ChatGLMConfig
41
30
-from .interfaces import SupportsLoRA, SupportsPP
31
-from .utils import (is_pp_missing_parameter,
32
- make_empty_intermediate_tensors_factory, make_layers)
42
+from .interfaces import SupportsLoRA, SupportsMultiModal
43
+
44
+logger = init_logger(__name__)
45
+
46
+
47
+def calculate_image_placeholder(vision_config):
48
+ return (vision_config["image_size"] // vision_config["patch_size"] // 2)**2
49
+
50
+
51
+def mm_input_mapper_for_glmv(
52
+ ctx: InputContext,
53
+ data: MultiModalData[object],
54
+) -> Dict:
55
+ model_config = ctx.model_config
56
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
57
+ trust_remote_code=True)
58
+ if tokenizer is None:
59
+ raise RuntimeError("No HuggingFace processor is available "
60
+ "to process the image object")
61
+ try:
62
+ raw_batch_data = tokenizer.apply_chat_template(
63
+ conversation=[{
64
+ "role": "user",
65
+ "image": data
66
+ }],
67
+ add_generation_prompt=True,
68
+ tokenize=True,
69
+ return_tensors="pt",
70
+ return_dict=True).data
71
+ except Exception:
72
+ logger.error("Failed to process image (%s)", data)
73
+ raise
74
+ pixel_values = raw_batch_data['images']
75
+
76
+ return MultiModalInputs({'pixel_values': pixel_values})
77
+
78
+
79
+def merge_glm_vision_embeddings(
80
+ input_ids: torch.Tensor,
81
+ inputs_embeds: torch.Tensor,
82
+ vision_embeddings: torch.Tensor,
83
+ boi_token_id: int,
84
+ eoi_token_id: int,
85
+) -> torch.Tensor:
86
+
87
+ boi_positions = (input_ids == boi_token_id).nonzero(as_tuple=True)[0]
88
+ eoi_positions = (input_ids == eoi_token_id).nonzero(as_tuple=True)[0]
89
+
90
+ mask = torch.zeros_like(input_ids, dtype=torch.bool)
91
+
92
+ for boi_pos, eoi_pos in zip(boi_positions, eoi_positions):
93
+ assert boi_pos < eoi_pos
94
+ mask[boi_pos:eoi_pos + 1] = True
95
+ inputs_embeds[mask] = vision_embeddings.view(-1,
96
+ vision_embeddings.shape[-1])
97
+ return inputs_embeds
98
+
99
+
100
+class GLMImagePixelInputs(TypedDict):
101
+ pixel_values: torch.Tensor
102
+ """Shape: `(batch_size, num_channels, height, width)`"""
103
+
104
+
105
+def get_max_glmv_image_tokens(ctx: InputContext):
106
+ hf_config = ctx.get_hf_config(ChatGLMConfig)
107
+
108
+ vision_config = getattr(hf_config, 'vision_config', None)
109
+ if vision_config is None:
110
+ return 1
111
+ elif isinstance(vision_config, dict):
112
+ return calculate_image_placeholder(vision_config)
113
+
114
+ msg = f"Unsupported vision config: {type(vision_config)}"
115
+ raise NotImplementedError(msg)
116
+
117
+
118
+def dummy_data_for_glmv(
119
+ ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
120
+) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
121
+ hf_config = ctx.get_hf_config(ChatGLMConfig)
122
+ vision_config = getattr(hf_config, 'vision_config', None)
123
+
124
+ if vision_config is None:
125
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len)
126
+ seq_data = SequenceData(token_ids)
127
+ return seq_data, None
128
+ elif isinstance(vision_config, dict):
129
+ image_size = vision_config["image_size"]
130
+ image_placeholder_length = calculate_image_placeholder(vision_config)
131
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE, [hf_config.boi_token_id] +
132
+ [0] * image_placeholder_length +
133
+ [hf_config.eoi_token_id])
134
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
135
+ [0] * (seq_len - image_placeholder_length - 2))
136
+ seq_data = SequenceData(token_ids)
137
+
138
+ mm_data = {
139
+ "image": Image.new("RGB", (image_size, image_size), color=0)
140
+ }
141
+
142
+ return seq_data, mm_data
143
+
144
+ msg = f"Unsupported vision config: {type(vision_config)}"
145
+ raise NotImplementedError(msg)
146
+
147
+
148
+def find_all_positions(input_ids: List[int], target: int) -> List[int]:
149
+ return [index for index, value in enumerate(input_ids) if value == target]
150
+
151
+
152
+def input_processor_for_glmv(ctx: InputContext, llm_inputs: LLMInputs):
153
+ hf_config = ctx.get_hf_config(ChatGLMConfig)
154
+ vision_config = getattr(hf_config, 'vision_config', None)
155
+
156
+ if vision_config is None:
157
+ return llm_inputs
158
+ elif isinstance(vision_config, dict):
159
+ image_placeholder_length = calculate_image_placeholder(vision_config)
160
+ else:
161
+ msg = f"Unsupported vision config: {type(vision_config)}"
162
+ raise NotImplementedError(msg)
163
+
164
+ input_ids = llm_inputs.get("prompt_token_ids")
165
+ position_ids = llm_inputs.get("position_ids")
166
+ tokenizer = cached_get_tokenizer(
167
+ ctx.model_config.model,
168
+ trust_remote_code=ctx.model_config.trust_remote_code)
169
+
170
+ try:
171
+ raw_batch_data = tokenizer.apply_chat_template(
172
+ conversation=[{
173
+ "role": "user",
174
+ "image": llm_inputs['multi_modal_data']["image"],
175
+ "content": llm_inputs['prompt']
176
+ }],
177
+ add_generation_prompt=True,
178
+ tokenize=True,
179
+ return_tensors="pt",
180
+ return_dict=True).data
181
+ except Exception:
182
+ logger.error("Failed to process content (%s)", llm_inputs['prompt'])
183
+ raise
184
+ input_ids = raw_batch_data['input_ids'][0].tolist()
185
+
186
+ if position_ids is None:
187
+ position_ids = list(range(len(input_ids)))
188
+ boi_token_id = hf_config.boi_token_id
189
+ eoi_token_id = hf_config.eoi_token_id
190
+ boi_positions = find_all_positions(input_ids, boi_token_id)
191
+ eoi_positions = find_all_positions(input_ids, eoi_token_id)
192
+
193
+ assert len(boi_positions) == len(eoi_positions)
194
+
195
+ new_input_ids = []
196
+ new_position_ids = []
197
+ final_processed_position = 0
198
+ final_processed_position = 0
199
+
200
+ for boi_position, eoi_position in zip(boi_positions, eoi_positions):
201
+ assert boi_position < eoi_position
202
+ new_input_ids.extend(input_ids[final_processed_position:boi_position +
203
+ 1])
204
+ new_position_ids.extend(
205
+ list(range(final_processed_position, boi_position + 1)))
206
+ new_input_ids.extend([input_ids[boi_position + 1]] *
207
+ image_placeholder_length)
208
+ new_position_ids.extend([boi_position + 1] * image_placeholder_length)
209
+ final_processed_position = eoi_position
210
+
211
+ new_input_ids.extend(input_ids[final_processed_position:])
212
+ new_position_ids.extend(
213
+ list(range(final_processed_position, len(input_ids))))
214
+
215
+ assert len(new_input_ids) == len(new_position_ids)
216
+
217
+ llm_inputs["prompt_token_ids"] = new_input_ids
218
+ llm_inputs["position_ids"] = new_position_ids
219
+ return llm_inputs
220
221
222
class GLMAttention(nn.Module):
223
224
def __init__(
225
self,
39
- config: ChatGLMConfig,
226
+ config,
227
cache_config: Optional[CacheConfig] = None,
228
quant_config: Optional[QuantizationConfig] = None,
229
):
@@ -127,7 +314,7 @@ class GLMMLP(nn.Module):
314
315
def __init__(
316
self,
130
- config: ChatGLMConfig,
317
+ config,
318
quant_config: Optional[QuantizationConfig] = None,
319
):
320
super().__init__()
@@ -170,7 +357,7 @@ class GLMBlock(nn.Module):
357
358
def __init__(
359
self,
173
- config: ChatGLMConfig,
360
+ config,
361
cache_config: Optional[CacheConfig] = None,
362
quant_config: Optional[QuantizationConfig] = None,
363
):
@@ -241,10 +428,9 @@ class GLMTransformer(nn.Module):
428
429
def __init__(
430
self,
244
- config: ChatGLMConfig,
431
+ config,
432
cache_config: Optional[CacheConfig] = None,
433
quant_config: Optional[QuantizationConfig] = None,
247
- prefix: str = "",
434
):
435
super().__init__()
436
self.post_layer_norm = config.post_layer_norm
@@ -253,11 +439,10 @@ class GLMTransformer(nn.Module):
439
self.num_layers = config.num_layers
440
441
# Transformer layers.
256
- self.start_layer, self.end_layer, self.layers = make_layers(
257
- self.num_layers,
258
- lambda prefix: GLMBlock(config, cache_config, quant_config),
259
- prefix=f"{prefix}.layers",
260
- )
442
+ self.layers = nn.ModuleList([
443
+ GLMBlock(config, cache_config, quant_config)
444
+ for i in range(self.num_layers)
445
+ ])
446
447
if self.post_layer_norm:
448
layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
@@ -272,16 +457,16 @@ class GLMTransformer(nn.Module):
457
kv_caches: List[torch.Tensor],
458
attn_metadata: AttentionMetadata,
459
) -> torch.Tensor:
275
- for i in range(self.start_layer, self.end_layer):
460
+ for i in range(self.num_layers):
461
layer = self.layers[i]
462
hidden_states = layer(
463
hidden_states=hidden_states,
464
position_ids=position_ids,
280
- kv_cache=kv_caches[i - self.start_layer],
465
+ kv_cache=kv_caches[i],
466
attn_metadata=attn_metadata,
467
)
468
# Final layer norm.
284
- if get_pp_group().is_last_rank and self.post_layer_norm:
469
+ if self.post_layer_norm:
470
hidden_states = self.final_layernorm(hidden_states)
471
472
return hidden_states
@@ -291,14 +476,17 @@ class ChatGLMModel(nn.Module):
476
477
def __init__(
478
self,
294
- config: ChatGLMConfig,
479
+ config,
480
cache_config: Optional[CacheConfig] = None,
481
quant_config: Optional[QuantizationConfig] = None,
482
):
483
super().__init__()
484
485
+ self.config = config
486
+
487
self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
301
- config.hidden_size)
488
+ config.hidden_size,
489
+ quant_config=quant_config)
490
491
self.num_layers = config.num_layers
492
self.multi_query_group_num = config.multi_query_group_num
@@ -308,37 +496,73 @@ class ChatGLMModel(nn.Module):
496
self.output_layer = ParallelLMHead(config.padded_vocab_size,
497
config.hidden_size,
498
quant_config=quant_config)
311
- self.make_empty_intermediate_tensors = (
312
- make_empty_intermediate_tensors_factory(["hidden_states"],
313
- config.hidden_size))
499
+
500
+ vision_config_flag = getattr(config, 'vision_config', None)
501
+ if vision_config_flag is not None:
502
+ self.vision_config = Namespace(**config.vision_config)
503
+ self.vision = EVA2CLIPModel(self.config, quant_config)
504
+ else:
505
+ self.vision = None
506
+
507
+ def _parse_and_validate_image_input(
508
+ self, **kwargs: object) -> GLMImagePixelInputs:
509
+
510
+ pixel_values = kwargs.pop("pixel_values", None)
511
+ if pixel_values is not None and self.vision is not None:
512
+ if isinstance(pixel_values, torch.Tensor):
513
+ if pixel_values.ndim > 2:
514
+ pixel_values = torch.concat(list(pixel_values))
515
+ elif isinstance(pixel_values, list):
516
+ return torch.concat(pixel_values)
517
+ else:
518
+ raise TypeError("""pixel_values must be a torch.Tensor
519
+ or a list of torch.Tensor
520
+ """)
521
+ return GLMImagePixelInputs(pixel_values=pixel_values)
522
523
def forward(
524
self,
525
input_ids: torch.Tensor,
318
- position_ids: torch.Tensor,
526
+ positions: torch.Tensor,
527
kv_caches: List[torch.Tensor],
528
attn_metadata: AttentionMetadata,
321
- intermediate_tensors: Optional[IntermediateTensors],
322
- ) -> Union[torch.Tensor, IntermediateTensors]:
323
- if get_pp_group().is_first_rank:
324
- inputs_embeds = self.embedding(input_ids)
325
- else:
326
- inputs_embeds = intermediate_tensors["hidden_states"]
529
+ intermediate_tensors: Optional[IntermediateTensors] = None,
530
+ **kwargs: object,
531
+ ) -> torch.Tensor:
532
+
533
+ inputs_embeds = self.embedding(input_ids)
534
+ image_input = self._parse_and_validate_image_input(**kwargs)
535
+
536
+ if image_input["pixel_values"] is not None:
537
+ pixel_values = image_input["pixel_values"].to(
538
+ dtype=inputs_embeds.dtype)
539
+ image_embeds = self.vision(pixel_values)
540
+
541
+ boi_token_id = self.config.boi_token_id
542
+ eoi_token_id = self.config.eoi_token_id
543
+
544
+ inputs_embeds = merge_glm_vision_embeddings(
545
+ input_ids=input_ids,
546
+ inputs_embeds=inputs_embeds,
547
+ vision_embeddings=image_embeds,
548
+ boi_token_id=boi_token_id,
549
+ eoi_token_id=eoi_token_id)
550
551
# Run encoder.
552
hidden_states = self.encoder(
553
hidden_states=inputs_embeds,
331
- position_ids=position_ids,
554
+ position_ids=positions,
555
kv_caches=kv_caches,
556
attn_metadata=attn_metadata,
557
)
335
-
336
- if not get_pp_group().is_last_rank:
337
- return IntermediateTensors({"hidden_states": hidden_states})
558
return hidden_states
559
560
341
-class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
561
+@MULTIMODAL_REGISTRY.register_image_input_mapper(mm_input_mapper_for_glmv)
562
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_glmv_image_tokens)
563
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_glmv)
564
+@INPUT_REGISTRY.register_input_processor(input_processor_for_glmv)
565
+class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
566
packed_modules_mapping = {
567
"query_key_value": ["query_key_value"],
568
"dense_h_to_4h": ["dense_h_to_4h"]
@@ -356,6 +580,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
580
def __init__(
581
self,
582
config: ChatGLMConfig,
583
+ multimodal_config: MultiModalConfig,
584
cache_config: Optional[CacheConfig] = None,
585
quant_config: Optional[QuantizationConfig] = None,
586
lora_config: Optional[LoRAConfig] = None,
@@ -364,6 +589,7 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
589
590
self.config = config
591
self.lora_config = lora_config
592
+ self.multimodal_config = multimodal_config
593
594
self.quant_config = quant_config
595
self.max_position_embeddings = getattr(config, "max_sequence_length",
@@ -375,19 +601,16 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
601
self.lm_head = self.transformer.output_layer
602
self.logits_processor = LogitsProcessor(config.padded_vocab_size)
603
self.sampler = Sampler()
378
- self.make_empty_intermediate_tensors = (
379
- self.transformer.make_empty_intermediate_tensors)
604
381
- def forward(
382
- self,
383
- input_ids: torch.Tensor,
384
- positions: torch.Tensor,
385
- kv_caches: List[torch.Tensor],
386
- attn_metadata: AttentionMetadata,
387
- intermediate_tensors: Optional[IntermediateTensors] = None,
388
- ) -> Union[torch.Tensor, IntermediateTensors]:
605
+ def forward(self,
606
+ input_ids: torch.Tensor,
607
+ positions: torch.Tensor,
608
+ kv_caches: List[torch.Tensor],
609
+ attn_metadata: AttentionMetadata,
610
+ intermediate_tensors: Optional[IntermediateTensors] = None,
611
+ **kwargs) -> torch.Tensor:
612
hidden_states = self.transformer(input_ids, positions, kv_caches,
390
- attn_metadata, intermediate_tensors)
613
+ attn_metadata, **kwargs)
614
return hidden_states
615
616
def compute_logits(
@@ -408,8 +631,24 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
631
return next_tokens
632
633
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
634
+ # Merge two ColumnParallelLinear into one MergedColumnParallelLinear
635
+ merged_weights_dict: Dict[str, Dict[str, Optional[torch.Tensor]]] = {
636
+ "transformer.vision.linear_proj.merged_proj.weight": {
637
+ "transformer.vision.linear_proj.gate_proj.weight": None,
638
+ "transformer.vision.linear_proj.dense_h_to_4h.weight": None,
639
+ }
640
+ }
641
+
642
params_dict = dict(self.named_parameters(remove_duplicate=False))
643
for name, loaded_weight in weights:
644
+ is_weight_to_be_merge = False
645
+ for _, merged_weight_dict in merged_weights_dict.items():
646
+ if name in merged_weight_dict:
647
+ assert merged_weight_dict[name] is None
648
+ merged_weight_dict[name] = loaded_weight
649
+ is_weight_to_be_merge = True
650
+ if is_weight_to_be_merge:
651
+ continue
652
if "rotary_pos_emb.inv_freq" in name:
653
continue
654
if "word_embeddings" in name:
@@ -417,9 +656,16 @@ class ChatGLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
656
# Skip loading extra bias for GPTQ models.
657
if name.endswith(".bias") and name not in params_dict:
658
continue
420
- if is_pp_missing_parameter(name, self):
421
- continue
659
param = params_dict[name]
660
weight_loader = getattr(param, "weight_loader",
661
default_weight_loader)
662
weight_loader(param, loaded_weight)
663
+
664
+ for combined_name, merged_weight_dict in merged_weights_dict.items():
665
+ if combined_name in params_dict:
666
+ param = params_dict[combined_name]
667
+ combined_weight = torch.cat(list(merged_weight_dict.values()),
668
+ dim=0)
669
+ weight_loader = getattr(param, "weight_loader",
670
+ default_weight_loader)
671
+ weight_loader(param, combined_weight)
vllm/model_executor/models/glm4_vision_encoder.py
new
+298
@@ -0,0 +1,298 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/THUDM/GLM-4
4
+"""Inference-only GLM-4v model visual encoder compatible with THUDM weights."""
5
+from argparse import Namespace
6
+from typing import Optional
7
+
8
+import torch
9
+from torch import nn
10
+from torch.nn import LayerNorm
11
+
12
+from vllm.distributed import get_tensor_model_parallel_world_size
13
+from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
14
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
15
+ MergedColumnParallelLinear,
16
+ QKVParallelLinear,
17
+ ReplicatedLinear,
18
+ RowParallelLinear)
19
+from vllm.model_executor.layers.quantization.base_config import (
20
+ QuantizationConfig)
21
+
22
+
23
+class PatchEmbedding(nn.Module):
24
+
25
+ def __init__(self, config):
26
+ super().__init__()
27
+ self.proj = nn.Conv2d(config.in_channels,
28
+ config.hidden_size,
29
+ kernel_size=config.patch_size,
30
+ stride=config.patch_size)
31
+ self.cls_embedding = nn.Parameter(torch.zeros(1, config.hidden_size))
32
+ self.position_embedding = nn.Embedding(config.num_positions,
33
+ config.hidden_size)
34
+
35
+ def forward(self, images: torch.Tensor) -> torch.Tensor:
36
+ """
37
+ Parameters:
38
+ images : torch.Tensor
39
+ Input image tensor with shape (B, C, H, W)
40
+
41
+ Returns:
42
+ torch.Tensor
43
+ Transformed tensor with shape (B, L, D)
44
+ """
45
+ images = images.to(self.proj.weight.device)
46
+ x = self.proj(images)
47
+ x = x.flatten(2).transpose(1, 2)
48
+ cls_token = self.cls_embedding.expand(x.shape[0], -1, -1)
49
+ x = torch.cat((cls_token, x), dim=1)
50
+ x += self.position_embedding.weight.unsqueeze(0)
51
+ return x
52
+
53
+
54
+class Attention(nn.Module):
55
+
56
+ def __init__(
57
+ self,
58
+ config,
59
+ quant_config: Optional[QuantizationConfig] = None,
60
+ ):
61
+ super().__init__()
62
+ self.hidden_size = config.hidden_size
63
+ self.tp_size = get_tensor_model_parallel_world_size()
64
+ self.num_heads_per_rank = config.num_heads // self.tp_size
65
+ self.head_dim = config.hidden_size // config.num_heads
66
+ self.scale = self.head_dim**-0.5
67
+
68
+ self.query_key_value = QKVParallelLinear(
69
+ config.hidden_size,
70
+ self.head_dim,
71
+ config.num_heads,
72
+ quant_config=quant_config,
73
+ )
74
+ self.dense = RowParallelLinear(
75
+ config.hidden_size,
76
+ config.hidden_size,
77
+ quant_config=quant_config,
78
+ )
79
+
80
+ self.output_dropout = torch.nn.Dropout(config.dropout_prob)
81
+
82
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
83
+ B, L, _ = x.shape
84
+ qkv, _ = self.query_key_value(x) # B, L, 3 * H * D
85
+ q, k, v = qkv.chunk(3, dim=-1)
86
+ q = q.reshape(B, L, self.num_heads_per_rank,
87
+ self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
88
+ k = k.reshape(B, L, self.num_heads_per_rank,
89
+ self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
90
+ v = v.reshape(B, L, self.num_heads_per_rank,
91
+ self.head_dim).permute(0, 2, 1, 3) # B, H, L, D
92
+
93
+ out = torch.nn.functional.scaled_dot_product_attention(q,
94
+ k,
95
+ v,
96
+ attn_mask=None,
97
+ dropout_p=0.,
98
+ is_causal=False)
99
+
100
+ output, _ = self.dense(out.transpose(1, 2).view(B, L, -1))
101
+ output = self.output_dropout(output)
102
+ return output
103
+
104
+
105
+class MLP(nn.Module):
106
+
107
+ def __init__(
108
+ self,
109
+ config,
110
+ quant_config: Optional[QuantizationConfig] = None,
111
+ ):
112
+ super().__init__()
113
+ self.config = config
114
+ self.activation_fn = get_act_fn(config.hidden_act)
115
+ self.fc1 = ColumnParallelLinear(
116
+ config.hidden_size,
117
+ config.intermediate_size,
118
+ quant_config=quant_config,
119
+ )
120
+ self.fc2 = RowParallelLinear(
121
+ config.intermediate_size,
122
+ config.hidden_size,
123
+ quant_config=quant_config,
124
+ )
125
+
126
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
127
+ x, _ = self.fc1(x)
128
+ x = self.activation_fn(x)
129
+ x, _ = self.fc2(x)
130
+ return x
131
+
132
+
133
+class TransformerLayer(nn.Module):
134
+
135
+ def __init__(
136
+ self,
137
+ config,
138
+ quant_config: Optional[QuantizationConfig] = None,
139
+ ):
140
+ super().__init__()
141
+ self.input_layernorm = LayerNorm(config.hidden_size,
142
+ eps=config.layer_norm_eps)
143
+ self.attention = Attention(config, quant_config=quant_config)
144
+ self.mlp = MLP(config, quant_config=quant_config)
145
+ self.post_attention_layernorm = LayerNorm(config.hidden_size,
146
+ eps=config.layer_norm_eps)
147
+
148
+ def forward(self, hidden_states):
149
+ attention_input = hidden_states
150
+ attention_output = self.input_layernorm(
151
+ self.attention(attention_input))
152
+ hidden_states = attention_input + attention_output
153
+ mlp_input = hidden_states
154
+ mlp_output = self.post_attention_layernorm(self.mlp(mlp_input))
155
+ output = mlp_input + mlp_output
156
+ return output
157
+
158
+
159
+class Transformer(nn.Module):
160
+
161
+ def __init__(
162
+ self,
163
+ config,
164
+ quant_config: Optional[QuantizationConfig] = None,
165
+ ):
166
+ super().__init__()
167
+ self.layers = nn.ModuleList([
168
+ TransformerLayer(config, quant_config=quant_config)
169
+ for _ in range(config.num_hidden_layers)
170
+ ])
171
+
172
+ def forward(self, hidden_states):
173
+ for layer_module in self.layers:
174
+ hidden_states = layer_module(hidden_states)
175
+ return hidden_states
176
+
177
+
178
+class GLU(nn.Module):
179
+
180
+ def __init__(
181
+ self,
182
+ config,
183
+ in_features,
184
+ quant_config: Optional[QuantizationConfig] = None,
185
+ ):
186
+ """
187
+ The original implementation is the same as:
188
+ ```python
189
+ self.dense_h_to_4h = ColumnParallelLinear(
190
+ config.hidden_size,
191
+ config.ffn_hidden_size,
192
+ bias=False,
193
+ quant_config=quant_config
194
+ )
195
+
196
+ self.gate_proj = ColumnParallelLinear(
197
+ config.hidden_size,
198
+ config.ffn_hidden_size,
199
+ bias=False,
200
+ quant_config=quant_config
201
+ )
202
+ ```
203
+ ```
204
+ gate_proj_output, _ = self.gate_proj(x)
205
+ dense_h_to_4h_output, _ = self.dense_h_to_4h(x)
206
+ x = torch.cat([gate_proj_output, dense_h_to_4h_output], dim=-1)
207
+ ```
208
+
209
+ We merge two ColumnParallelLinear into one MergedColumnParallelLinear:
210
+ ```
211
+ self.merged_proj = MergedColumnParallelLinear(
212
+ config.hidden_size,
213
+ [config.ffn_hidden_size] * 2,
214
+ bias=False,
215
+ quant_config=quant_config
216
+ )
217
+ ```
218
+ ```
219
+ x, _ = self.merged_proj(x)
220
+ ```
221
+ """
222
+ super().__init__()
223
+ self.linear_proj = ReplicatedLinear(in_features,
224
+ config.hidden_size,
225
+ bias=False,
226
+ quant_config=quant_config)
227
+ self.norm1 = nn.LayerNorm(config.hidden_size)
228
+ self.act1 = nn.GELU()
229
+ self.act2 = SiluAndMul()
230
+
231
+ self.merged_proj = MergedColumnParallelLinear(
232
+ config.hidden_size, [config.ffn_hidden_size] * 2,
233
+ bias=False,
234
+ quant_config=quant_config)
235
+
236
+ self.dense_4h_to_h = RowParallelLinear(config.ffn_hidden_size,
237
+ config.hidden_size,
238
+ bias=False,
239
+ quant_config=quant_config)
240
+
241
+ def forward(self, x):
242
+ x, _ = self.linear_proj(x)
243
+ x = self.act1(self.norm1(x))
244
+ x, _ = self.merged_proj(x)
245
+ x = self.act2(x)
246
+ x, _ = self.dense_4h_to_h(x)
247
+ return x
248
+
249
+
250
+class EVA2CLIPModel(nn.Module):
251
+
252
+ def __init__(
253
+ self,
254
+ config,
255
+ quant_config: Optional[QuantizationConfig] = None,
256
+ ):
257
+ super().__init__()
258
+ vision_config = Namespace(**config.vision_config)
259
+ self.patch_embedding = PatchEmbedding(vision_config)
260
+ self.transformer = Transformer(vision_config,
261
+ quant_config=quant_config)
262
+ self.linear_proj = GLU(config,
263
+ in_features=config.hidden_size,
264
+ quant_config=quant_config)
265
+ self.conv = nn.Conv2d(in_channels=vision_config.hidden_size,
266
+ out_channels=config.hidden_size,
267
+ kernel_size=2,
268
+ stride=2)
269
+ self.boi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
270
+ self.eoi = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
271
+ self.scaling_factor = vision_config.scaling_factor
272
+
273
+ def forward(self, images: torch.Tensor) -> torch.Tensor:
274
+ """
275
+ Parameters:
276
+ images : torch.Tensor
277
+ Input image tensor with shape (B, C, H, W)
278
+
279
+ Returns:
280
+ torch.Tensor
281
+ Transformed tensor with shape (B, L, D)
282
+ """
283
+ x = self.patch_embedding(images)
284
+ x = self.transformer(x)
285
+ x = x[:, 1:]
286
+
287
+ b, s, h = x.shape
288
+ grid_size = int(s**0.5)
289
+ x = x.view(b, grid_size, grid_size, h).permute(0, 3, 1, 2)
290
+ x = self.conv(x)
291
+
292
+ x = x.flatten(2).transpose(1, 2)
293
+ x = self.linear_proj(x)
294
+ boi = self.boi.expand(x.shape[0], -1, -1)
295
+ eoi = self.eoi.expand(x.shape[0], -1, -1)
296
+ x = torch.cat((boi, x, eoi), dim=1)
297
+ x = x / self.scaling_factor
298
+ return x
vllm/model_executor/models/registry.py
+4
-2
@@ -29,8 +29,7 @@ _TEXT_GENERATION_MODELS = {
29
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
30
"BaichuanForCausalLM": ("baichuan", "BaichuanForCausalLM"), # baichuan-13b
31
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
32
- "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
33
- "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
32
+ # ChatGLMModel supports multimodal
33
"CohereForCausalLM": ("commandr", "CohereForCausalLM"),
34
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
35
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
@@ -72,6 +71,7 @@ _TEXT_GENERATION_MODELS = {
71
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
72
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
73
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
74
+ # QWenLMHeadModel supports multimodal
75
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
76
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
77
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
@@ -95,6 +95,8 @@ _MULTIMODAL_MODELS = {
95
# [Decoder-only]
96
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
97
"ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
98
+ "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
99
+ "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
100
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
101
"InternVLChatModel": ("internvl", "InternVLChatModel"),
102
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
vllm/transformers_utils/tokenizer.py
+21
-18
@@ -59,6 +59,26 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
59
return tokenizer
60
61
62
+def patch_padding_side(tokenizer: PreTrainedTokenizer) -> None:
63
+ """Patch _pad method to accept `padding_side` for older tokenizers."""
64
+ orig_pad = tokenizer._pad
65
+
66
+ def _pad(
67
+ self: PreTrainedTokenizer,
68
+ *args,
69
+ padding_side: Optional[str] = None,
70
+ **kwargs,
71
+ ):
72
+ if padding_side is not None and padding_side != self.padding_side:
73
+ msg = ("`padding_side` argument is not supported by "
74
+ f"{type(tokenizer).__name__} and will be ignored.")
75
+ warnings.warn(msg, stacklevel=2)
76
+
77
+ return orig_pad(*args, **kwargs)
78
+
79
+ tokenizer._pad = MethodType(_pad, tokenizer)
80
+
81
+
82
def get_tokenizer(
83
tokenizer_name: Union[str, Path],
84
*args,
@@ -143,24 +163,7 @@ def get_tokenizer(
163
if type(tokenizer).__name__ in ("ChatGLMTokenizer",
164
"ChatGLM4Tokenizer"):
165
assert isinstance(tokenizer, PreTrainedTokenizer)
146
- orig_pad = tokenizer._pad
147
-
148
- # Patch _pad method to accept `padding_side`
149
- def _pad(
150
- self: PreTrainedTokenizer,
151
- *args,
152
- padding_side: Optional[str] = None,
153
- **kwargs,
154
- ):
155
- if (padding_side is not None
156
- and padding_side != self.padding_side):
157
- msg = ("`padding_side` argument is not supported by "
158
- "ChatGLMTokenizer and will be ignored.")
159
- warnings.warn(msg, stacklevel=2)
160
-
161
- return orig_pad(*args, **kwargs)
162
-
163
- tokenizer._pad = MethodType(_pad, tokenizer)
166
+ patch_padding_side(tokenizer)
167
168
if not isinstance(tokenizer, PreTrainedTokenizerFast):
169
logger.warning(