A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
ChatGLM Support (#1261)
GoHomeToMacDonal committed
Nov 7, 2023 at 08:09 UTC
1a2bbc930135cd3b94fbff2aafbdf5c568acc8bd
7 files changed
+490
-4
vllm/config.py
+4
@@ -166,6 +166,10 @@ class ModelConfig:
166
if getattr(self.hf_config, "num_key_value_heads", None) is not None:
167
return (self.hf_config.num_key_value_heads //
168
parallel_config.tensor_parallel_size)
169
+ # For ChatGLM-2:
170
+ if getattr(self.hf_config, "multi_query_group_num", None) is not None:
171
+ return (self.hf_config.multi_query_group_num //
172
+ parallel_config.tensor_parallel_size)
173
total_num_attention_heads = self.hf_config.num_attention_heads
174
return total_num_attention_heads // parallel_config.tensor_parallel_size
175
vllm/model_executor/model_loader.py
+1
@@ -18,6 +18,7 @@ _MODEL_REGISTRY = {
18
"BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
19
"BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
20
"BloomForCausalLM": BloomForCausalLM,
21
+ "ChatGLMModel": ChatGLMForCausalLM,
22
"FalconForCausalLM": FalconForCausalLM,
23
"GPT2LMHeadModel": GPT2LMHeadModel,
24
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.model_executor.models.mistral import MistralForCausalLM
13
from vllm.model_executor.models.mpt import MptForCausalLM
14
from vllm.model_executor.models.opt import OPTForCausalLM
15
from vllm.model_executor.models.qwen import QWenLMHeadModel
16
+from vllm.model_executor.models.chatglm import ChatGLMForCausalLM
17
from vllm.model_executor.models.yi import YiForCausalLM
18
19
__all__ = [
@@ -20,6 +21,7 @@ __all__ = [
21
"BaiChuanForCausalLM",
22
"BaichuanForCausalLM",
23
"BloomForCausalLM",
24
+ "ChatGLMForCausalLM",
25
"FalconForCausalLM",
26
"GPT2LMHeadModel",
27
"GPTBigCodeForCausalLM",
vllm/model_executor/models/chatglm.py
new
+408
@@ -0,0 +1,408 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/THUDM/ChatGLM2-6B
4
+"""Inference-only ChatGLM model compatible with THUDM weights.
5
+
6
+The input of the model is flattened to a 1D tensor of tokens. The model uses
7
+InputMetadata to extract the original 2D shape of the input.
8
+"""
9
+from typing import Dict, List, Optional, Tuple
10
+
11
+import torch
12
+from torch import nn
13
+from torch.nn import LayerNorm
14
+
15
+from vllm.model_executor.input_metadata import InputMetadata
16
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
17
+from vllm.model_executor.layers.layernorm import RMSNorm
18
+from vllm.model_executor.layers.activation import SiluAndMul
19
+from vllm.model_executor.layers.sampler import Sampler
20
+from vllm.model_executor.weight_utils import (
21
+ hf_model_weights_iterator,
22
+ load_tensor_parallel_weights,
23
+)
24
+from vllm.model_executor.parallel_utils.parallel_state import (
25
+ get_tensor_model_parallel_rank,
26
+ get_tensor_model_parallel_world_size,
27
+)
28
+from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
29
+from vllm.model_executor.parallel_utils.layers import (
30
+ ColumnParallelLinear,
31
+ RowParallelLinear,
32
+)
33
+from vllm.sequence import SequenceOutputs
34
+
35
+from vllm.transformers_utils.configs import ChatGLMConfig
36
+
37
+KVCache = Tuple[torch.Tensor, torch.Tensor]
38
+
39
+
40
+class GLMAttention(nn.Module):
41
+
42
+ def __init__(self, config):
43
+ super().__init__()
44
+ self.hidden_size = config.hidden_size
45
+ tp_size = get_tensor_model_parallel_world_size()
46
+ self.total_num_heads = config.num_attention_heads
47
+ assert self.total_num_heads % tp_size == 0
48
+ self.num_heads = self.total_num_heads // tp_size
49
+ self.multi_query_attention = config.multi_query_attention
50
+ self.total_num_kv_heads = (config.multi_query_group_num
51
+ if config.multi_query_attention else
52
+ config.num_attention_heads)
53
+ assert self.total_num_kv_heads % tp_size == 0
54
+ self.num_kv_heads = self.total_num_kv_heads // tp_size
55
+ self.head_dim = config.hidden_size // self.total_num_heads
56
+ self.q_size = self.num_heads * self.head_dim
57
+ self.kv_size = self.num_kv_heads * self.head_dim
58
+ self.scaling = self.head_dim**-0.5
59
+
60
+ self.query_key_value = ColumnParallelLinear(
61
+ config.hidden_size,
62
+ (self.total_num_heads + 2 * self.total_num_kv_heads) *
63
+ self.head_dim,
64
+ bias=config.add_qkv_bias,
65
+ gather_output=False,
66
+ )
67
+ self.dense = RowParallelLinear(
68
+ self.total_num_heads * self.head_dim,
69
+ config.hidden_size,
70
+ bias=config.add_bias_linear,
71
+ input_is_parallel=True,
72
+ )
73
+
74
+ self.attn = PagedAttentionWithRoPE(
75
+ self.num_heads,
76
+ self.head_dim,
77
+ self.scaling,
78
+ rotary_dim=self.head_dim // 2,
79
+ num_kv_heads=self.num_kv_heads,
80
+ is_neox_style=False,
81
+ # is_glm_style=True
82
+ )
83
+
84
+ def forward(
85
+ self,
86
+ hidden_states: torch.Tensor,
87
+ position_ids: torch.Tensor,
88
+ kv_cache: KVCache,
89
+ input_metadata: InputMetadata,
90
+ cache_event: Optional[torch.cuda.Event],
91
+ ) -> torch.Tensor:
92
+ qkv, _ = self.query_key_value(hidden_states)
93
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
94
+ key_cache, value_cache = kv_cache
95
+
96
+ context_layer = self.attn(
97
+ position_ids,
98
+ q,
99
+ k,
100
+ v,
101
+ key_cache,
102
+ value_cache,
103
+ input_metadata,
104
+ cache_event,
105
+ )
106
+
107
+ attn_output, _ = self.dense(context_layer)
108
+
109
+ return attn_output
110
+
111
+
112
+class GLMMLP(nn.Module):
113
+ """MLP.
114
+
115
+ MLP will take the input with h hidden state, project it to 4*h
116
+ hidden dimension, perform nonlinear transformation, and project the
117
+ state back into h hidden dimension.
118
+ """
119
+
120
+ def __init__(self, config):
121
+ super().__init__()
122
+
123
+ self.add_bias = config.add_bias_linear
124
+
125
+ # Project to 4h.
126
+ self.dense_h_to_4h = ColumnParallelLinear(
127
+ config.hidden_size,
128
+ config.ffn_hidden_size * 2,
129
+ bias=config.add_bias_linear,
130
+ gather_output=False,
131
+ )
132
+
133
+ self.activation_func = SiluAndMul()
134
+
135
+ # Project back to h.
136
+ self.dense_4h_to_h = RowParallelLinear(
137
+ config.ffn_hidden_size,
138
+ config.hidden_size,
139
+ bias=config.add_bias_linear,
140
+ input_is_parallel=True,
141
+ )
142
+
143
+ def forward(self, hidden_states):
144
+ # [s, b, 4hp]
145
+ intermediate_parallel, _ = self.dense_h_to_4h(hidden_states)
146
+ intermediate_parallel = self.activation_func(intermediate_parallel)
147
+ # [s, b, h]
148
+ output, _ = self.dense_4h_to_h(intermediate_parallel)
149
+ return output
150
+
151
+
152
+class GLMBlock(nn.Module):
153
+ """A single transformer layer.
154
+
155
+ Transformer layer takes input with size [s, b, h] and returns an
156
+ output of the same size.
157
+ """
158
+
159
+ def __init__(
160
+ self,
161
+ config,
162
+ ):
163
+ super().__init__()
164
+ self.apply_residual_connection_post_layernorm = (
165
+ config.apply_residual_connection_post_layernorm)
166
+
167
+ self.fp32_residual_connection = config.fp32_residual_connection
168
+
169
+ layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
170
+ # Layernorm on the input data.
171
+ self.input_layernorm = layer_norm_func(config.hidden_size,
172
+ eps=config.layernorm_epsilon)
173
+
174
+ # Self attention.
175
+ self.self_attention = GLMAttention(config)
176
+ self.hidden_dropout = config.hidden_dropout
177
+
178
+ # Layernorm on the attention output
179
+ self.post_attention_layernorm = layer_norm_func(
180
+ config.hidden_size, eps=config.layernorm_epsilon)
181
+
182
+ # MLP
183
+ self.mlp = GLMMLP(config)
184
+
185
+ def forward(
186
+ self,
187
+ hidden_states: torch.Tensor,
188
+ position_ids: torch.Tensor,
189
+ kv_cache: KVCache,
190
+ input_metadata: InputMetadata,
191
+ cache_event: Optional[torch.cuda.Event],
192
+ ) -> torch.Tensor:
193
+ # hidden_states: [num_tokens, h]
194
+ # Layer norm at the beginning of the transformer layer.
195
+ layernorm_output = self.input_layernorm(hidden_states)
196
+ # Self attention.
197
+ attention_output = self.self_attention(
198
+ hidden_states=layernorm_output,
199
+ position_ids=position_ids,
200
+ kv_cache=kv_cache,
201
+ input_metadata=input_metadata,
202
+ cache_event=cache_event,
203
+ )
204
+
205
+ # Residual connection.
206
+ if self.apply_residual_connection_post_layernorm:
207
+ residual = layernorm_output
208
+ else:
209
+ residual = hidden_states
210
+
211
+ layernorm_input = residual + attention_output
212
+
213
+ # Layer norm post the self attention.
214
+ layernorm_output = self.post_attention_layernorm(layernorm_input)
215
+
216
+ # Second residual connection.
217
+ if self.apply_residual_connection_post_layernorm:
218
+ residual = layernorm_output
219
+ else:
220
+ residual = layernorm_input
221
+
222
+ output = self.mlp(layernorm_output) + residual
223
+
224
+ return output
225
+
226
+
227
+class GLMTransformer(nn.Module):
228
+ """Transformer class."""
229
+
230
+ def __init__(self, config):
231
+ super().__init__()
232
+ self.post_layer_norm = config.post_layer_norm
233
+
234
+ # Number of layers.
235
+ self.num_layers = config.num_layers
236
+
237
+ # Transformer layers.
238
+ self.layers = nn.ModuleList(
239
+ [GLMBlock(config) for i in range(self.num_layers)])
240
+
241
+ if self.post_layer_norm:
242
+ layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
243
+ # Final layer norm before output.
244
+ self.final_layernorm = layer_norm_func(
245
+ config.hidden_size, eps=config.layernorm_epsilon)
246
+
247
+ def forward(
248
+ self,
249
+ hidden_states: torch.Tensor,
250
+ position_ids: torch.Tensor,
251
+ kv_caches: List[KVCache],
252
+ input_metadata: InputMetadata,
253
+ cache_events: Optional[List[torch.cuda.Event]],
254
+ ) -> torch.Tensor:
255
+ for i in range(self.num_layers):
256
+ if cache_events is None:
257
+ cache_event = None
258
+ else:
259
+ cache_event = cache_events[i]
260
+ layer = self.layers[i]
261
+ hidden_states = layer(
262
+ hidden_states=hidden_states,
263
+ position_ids=position_ids,
264
+ kv_cache=kv_caches[i],
265
+ input_metadata=input_metadata,
266
+ cache_event=cache_event,
267
+ )
268
+ # Final layer norm.
269
+ if self.post_layer_norm:
270
+ hidden_states = self.final_layernorm(hidden_states)
271
+
272
+ return hidden_states
273
+
274
+
275
+class ChatGLMModel(nn.Module):
276
+
277
+ def __init__(self, config):
278
+ super().__init__()
279
+
280
+ self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
281
+ config.hidden_size)
282
+
283
+ self.num_layers = config.num_layers
284
+ self.multi_query_group_num = config.multi_query_group_num
285
+ self.kv_channels = config.kv_channels
286
+ self.encoder = GLMTransformer(config)
287
+
288
+ self.output_layer = ColumnParallelLinear(
289
+ config.hidden_size,
290
+ config.padded_vocab_size,
291
+ bias=False,
292
+ gather_output=False,
293
+ params_dtype=config.torch_dtype,
294
+ )
295
+
296
+ def forward(
297
+ self,
298
+ input_ids: torch.Tensor,
299
+ position_ids: torch.Tensor,
300
+ kv_caches: List[KVCache],
301
+ input_metadata: InputMetadata,
302
+ cache_events: Optional[List[torch.cuda.Event]],
303
+ ):
304
+ inputs_embeds = self.embedding(input_ids)
305
+
306
+ # Run encoder.
307
+ hidden_states = self.encoder(
308
+ hidden_states=inputs_embeds,
309
+ position_ids=position_ids,
310
+ kv_caches=kv_caches,
311
+ input_metadata=input_metadata,
312
+ cache_events=cache_events,
313
+ )
314
+
315
+ return hidden_states
316
+
317
+
318
+class ChatGLMForCausalLM(nn.Module):
319
+
320
+ def __init__(self, config: ChatGLMConfig):
321
+ super().__init__()
322
+ self.config: ChatGLMConfig = config
323
+ self.transformer = ChatGLMModel(config)
324
+ self.lm_head_weight = self.transformer.output_layer.weight
325
+ self.sampler = Sampler(config.padded_vocab_size)
326
+
327
+ def forward(
328
+ self,
329
+ input_ids: torch.Tensor,
330
+ positions: torch.Tensor,
331
+ kv_caches: List[KVCache],
332
+ input_metadata: InputMetadata,
333
+ cache_events: Optional[List[torch.cuda.Event]],
334
+ ) -> Dict[int, SequenceOutputs]:
335
+ hidden_states = self.transformer(input_ids, positions, kv_caches,
336
+ input_metadata, cache_events)
337
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
338
+ input_metadata)
339
+ return next_tokens
340
+
341
+ _column_parallel_weights = [
342
+ "output_layer.weight",
343
+ "embedding.weight",
344
+ ]
345
+ _row_parallel_weights = ["dense_4h_to_h", "self_attention.dense"]
346
+
347
+ def load_weights(
348
+ self,
349
+ model_name_or_path: str,
350
+ cache_dir: Optional[str] = None,
351
+ load_format: str = "auto",
352
+ revision: Optional[str] = None,
353
+ ):
354
+ tp_rank = get_tensor_model_parallel_rank()
355
+ tp_size = get_tensor_model_parallel_world_size()
356
+
357
+ q_proj_shard_size = self.config.hidden_size // tp_size
358
+ kv_proj_shard_size = (self.config.hidden_size //
359
+ self.config.num_attention_heads *
360
+ self.config.multi_query_group_num // tp_size)
361
+
362
+ mlp_hidden_shard_size = self.config.ffn_hidden_size // tp_size
363
+
364
+ state_dict = self.state_dict()
365
+ for name, loaded_weight in hf_model_weights_iterator(
366
+ model_name_or_path, cache_dir, load_format, revision):
367
+ if "word_embeddings" in name:
368
+ name = name.replace(".word_embeddings", "")
369
+
370
+ if name in state_dict:
371
+ param = state_dict[name]
372
+ if "query_key_value" in name:
373
+ q_offset = q_proj_shard_size * tp_rank
374
+ k_offset = (q_proj_shard_size * tp_size +
375
+ kv_proj_shard_size * tp_rank)
376
+ v_offset = (q_proj_shard_size * tp_size +
377
+ kv_proj_shard_size * (tp_size + tp_rank))
378
+ wq = loaded_weight[q_offset:q_offset + q_proj_shard_size]
379
+ wk = loaded_weight[k_offset:k_offset + kv_proj_shard_size]
380
+ wv = loaded_weight[v_offset:v_offset + kv_proj_shard_size]
381
+ loaded_weight = torch.cat([wq, wk, wv], dim=0)
382
+ param.data.copy_(loaded_weight)
383
+ continue
384
+
385
+ if "dense_h_to_4h" in name:
386
+ w_gate = loaded_weight[mlp_hidden_shard_size *
387
+ tp_rank:mlp_hidden_shard_size *
388
+ (tp_rank + 1)]
389
+ w_proj = loaded_weight[mlp_hidden_shard_size *
390
+ (tp_size +
391
+ tp_rank):mlp_hidden_shard_size *
392
+ (tp_size + tp_rank + 1)]
393
+ loaded_weight = torch.cat([w_gate, w_proj], dim=0)
394
+ param.data.copy_(loaded_weight)
395
+ continue
396
+
397
+ load_tensor_parallel_weights(
398
+ param,
399
+ loaded_weight,
400
+ name,
401
+ self._column_parallel_weights,
402
+ self._row_parallel_weights,
403
+ tp_rank,
404
+ )
405
+ elif name == "transformer.rotary_pos_emb.inv_freq":
406
+ continue
407
+ else:
408
+ print("Warning never found tensor's name:", name)
vllm/transformers_utils/config.py
+3
-2
@@ -5,9 +5,10 @@ from transformers import AutoConfig, MptConfig, PretrainedConfig
5
from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
6
7
_CONFIG_REGISTRY = {
8
- "mpt": MptConfig,
9
- "baichuan": BaiChuanConfig,
8
"aquila": AquilaConfig,
9
+ "baichuan": BaiChuanConfig,
10
+ "chatglm": ChatGLMConfig,
11
+ "mpt": MptConfig,
12
"qwen": QWenConfig,
13
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
14
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+4
-2
@@ -1,5 +1,6 @@
1
-from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
1
from vllm.transformers_utils.configs.aquila import AquilaConfig
2
+from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3
+from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4
from vllm.transformers_utils.configs.qwen import QWenConfig
5
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
6
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
@@ -8,8 +9,9 @@ from vllm.transformers_utils.configs.falcon import RWConfig
9
from vllm.transformers_utils.configs.yi import YiConfig
10
11
__all__ = [
11
- "BaiChuanConfig",
12
"AquilaConfig",
13
+ "BaiChuanConfig",
14
+ "ChatGLMConfig",
15
"QWenConfig",
16
"RWConfig",
17
"YiConfig",
vllm/transformers_utils/configs/chatglm.py
new
+68
@@ -0,0 +1,68 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/THUDM/ChatGLM2-6B
4
+from transformers import PretrainedConfig
5
+
6
+
7
+class ChatGLMConfig(PretrainedConfig):
8
+ model_type = "chatglm"
9
+ attribute_map = {
10
+ "num_hidden_layers": "num_layers",
11
+ "n_head_kv": "multi_query_group_num",
12
+ }
13
+
14
+ def __init__(self,
15
+ num_layers=28,
16
+ padded_vocab_size=65024,
17
+ hidden_size=4096,
18
+ ffn_hidden_size=13696,
19
+ kv_channels=128,
20
+ num_attention_heads=32,
21
+ seq_length=2048,
22
+ hidden_dropout=0.0,
23
+ attention_dropout=0.0,
24
+ layernorm_epsilon=1e-5,
25
+ rmsnorm=True,
26
+ apply_residual_connection_post_layernorm=False,
27
+ post_layer_norm=True,
28
+ add_bias_linear=False,
29
+ add_qkv_bias=False,
30
+ interleaved_qkv=False,
31
+ bias_dropout_fusion=True,
32
+ multi_query_attention=False,
33
+ multi_query_group_num=1,
34
+ apply_query_key_layer_scaling=True,
35
+ attention_softmax_in_fp32=True,
36
+ fp32_residual_connection=False,
37
+ quantization_bit=0,
38
+ pre_seq_len=None,
39
+ prefix_projection=False,
40
+ **kwargs):
41
+ self.num_layers = num_layers
42
+ self.vocab_size = padded_vocab_size
43
+ self.padded_vocab_size = padded_vocab_size
44
+ self.hidden_size = hidden_size
45
+ self.ffn_hidden_size = ffn_hidden_size
46
+ self.kv_channels = kv_channels
47
+ self.num_attention_heads = num_attention_heads
48
+ self.seq_length = seq_length
49
+ self.hidden_dropout = hidden_dropout
50
+ self.attention_dropout = attention_dropout
51
+ self.layernorm_epsilon = layernorm_epsilon
52
+ self.rmsnorm = rmsnorm
53
+ self.apply_residual_connection_post_layernorm = (
54
+ apply_residual_connection_post_layernorm)
55
+ self.post_layer_norm = post_layer_norm
56
+ self.add_bias_linear = add_bias_linear
57
+ self.add_qkv_bias = add_qkv_bias
58
+ self.bias_dropout_fusion = bias_dropout_fusion
59
+ self.multi_query_attention = multi_query_attention
60
+ self.multi_query_group_num = multi_query_group_num
61
+ self.apply_query_key_layer_scaling = apply_query_key_layer_scaling
62
+ self.attention_softmax_in_fp32 = attention_softmax_in_fp32
63
+ self.fp32_residual_connection = fp32_residual_connection
64
+ self.quantization_bit = quantization_bit
65
+ self.pre_seq_len = pre_seq_len
66
+ self.prefix_projection = prefix_projection
67
+ self.interleaved_qkv = interleaved_qkv
68
+ super().__init__(**kwargs)