@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a2bbc9301
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

ChatGLM Support (#1261)

GoHomeToMacDonal committed Nov 7, 2023 at 08:09 UTC 1a2bbc930135cd3b94fbff2aafbdf5c568acc8bd
7 files changed +490 -4
vllm/config.py
+4
@@ -166,6 +166,10 @@ class ModelConfig:
166 if getattr(self.hf_config, "num_key_value_heads", None) is not None:
167 return (self.hf_config.num_key_value_heads //
168 parallel_config.tensor_parallel_size)
169 + # For ChatGLM-2:
170 + if getattr(self.hf_config, "multi_query_group_num", None) is not None:
171 + return (self.hf_config.multi_query_group_num //
172 + parallel_config.tensor_parallel_size)
173 total_num_attention_heads = self.hf_config.num_attention_heads
174 return total_num_attention_heads // parallel_config.tensor_parallel_size
175
vllm/model_executor/model_loader.py
+1
@@ -18,6 +18,7 @@ _MODEL_REGISTRY = {
18 "BaiChuanForCausalLM": BaiChuanForCausalLM, # baichuan-7b
19 "BaichuanForCausalLM": BaichuanForCausalLM, # baichuan-13b
20 "BloomForCausalLM": BloomForCausalLM,
21 + "ChatGLMModel": ChatGLMForCausalLM,
22 "FalconForCausalLM": FalconForCausalLM,
23 "GPT2LMHeadModel": GPT2LMHeadModel,
24 "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
vllm/model_executor/models/__init__.py
+2
@@ -13,6 +13,7 @@ from vllm.model_executor.models.mistral import MistralForCausalLM
13 from vllm.model_executor.models.mpt import MptForCausalLM
14 from vllm.model_executor.models.opt import OPTForCausalLM
15 from vllm.model_executor.models.qwen import QWenLMHeadModel
16 +from vllm.model_executor.models.chatglm import ChatGLMForCausalLM
17 from vllm.model_executor.models.yi import YiForCausalLM
18
19 __all__ = [
@@ -20,6 +21,7 @@ __all__ = [
21 "BaiChuanForCausalLM",
22 "BaichuanForCausalLM",
23 "BloomForCausalLM",
24 + "ChatGLMForCausalLM",
25 "FalconForCausalLM",
26 "GPT2LMHeadModel",
27 "GPTBigCodeForCausalLM",
vllm/model_executor/models/chatglm.py new
+408
@@ -0,0 +1,408 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/THUDM/ChatGLM2-6B
4 +"""Inference-only ChatGLM model compatible with THUDM weights.
5 +
6 +The input of the model is flattened to a 1D tensor of tokens. The model uses
7 +InputMetadata to extract the original 2D shape of the input.
8 +"""
9 +from typing import Dict, List, Optional, Tuple
10 +
11 +import torch
12 +from torch import nn
13 +from torch.nn import LayerNorm
14 +
15 +from vllm.model_executor.input_metadata import InputMetadata
16 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
17 +from vllm.model_executor.layers.layernorm import RMSNorm
18 +from vllm.model_executor.layers.activation import SiluAndMul
19 +from vllm.model_executor.layers.sampler import Sampler
20 +from vllm.model_executor.weight_utils import (
21 + hf_model_weights_iterator,
22 + load_tensor_parallel_weights,
23 +)
24 +from vllm.model_executor.parallel_utils.parallel_state import (
25 + get_tensor_model_parallel_rank,
26 + get_tensor_model_parallel_world_size,
27 +)
28 +from vllm.model_executor.parallel_utils.layers import VocabParallelEmbedding
29 +from vllm.model_executor.parallel_utils.layers import (
30 + ColumnParallelLinear,
31 + RowParallelLinear,
32 +)
33 +from vllm.sequence import SequenceOutputs
34 +
35 +from vllm.transformers_utils.configs import ChatGLMConfig
36 +
37 +KVCache = Tuple[torch.Tensor, torch.Tensor]
38 +
39 +
40 +class GLMAttention(nn.Module):
41 +
42 + def __init__(self, config):
43 + super().__init__()
44 + self.hidden_size = config.hidden_size
45 + tp_size = get_tensor_model_parallel_world_size()
46 + self.total_num_heads = config.num_attention_heads
47 + assert self.total_num_heads % tp_size == 0
48 + self.num_heads = self.total_num_heads // tp_size
49 + self.multi_query_attention = config.multi_query_attention
50 + self.total_num_kv_heads = (config.multi_query_group_num
51 + if config.multi_query_attention else
52 + config.num_attention_heads)
53 + assert self.total_num_kv_heads % tp_size == 0
54 + self.num_kv_heads = self.total_num_kv_heads // tp_size
55 + self.head_dim = config.hidden_size // self.total_num_heads
56 + self.q_size = self.num_heads * self.head_dim
57 + self.kv_size = self.num_kv_heads * self.head_dim
58 + self.scaling = self.head_dim**-0.5
59 +
60 + self.query_key_value = ColumnParallelLinear(
61 + config.hidden_size,
62 + (self.total_num_heads + 2 * self.total_num_kv_heads) *
63 + self.head_dim,
64 + bias=config.add_qkv_bias,
65 + gather_output=False,
66 + )
67 + self.dense = RowParallelLinear(
68 + self.total_num_heads * self.head_dim,
69 + config.hidden_size,
70 + bias=config.add_bias_linear,
71 + input_is_parallel=True,
72 + )
73 +
74 + self.attn = PagedAttentionWithRoPE(
75 + self.num_heads,
76 + self.head_dim,
77 + self.scaling,
78 + rotary_dim=self.head_dim // 2,
79 + num_kv_heads=self.num_kv_heads,
80 + is_neox_style=False,
81 + # is_glm_style=True
82 + )
83 +
84 + def forward(
85 + self,
86 + hidden_states: torch.Tensor,
87 + position_ids: torch.Tensor,
88 + kv_cache: KVCache,
89 + input_metadata: InputMetadata,
90 + cache_event: Optional[torch.cuda.Event],
91 + ) -> torch.Tensor:
92 + qkv, _ = self.query_key_value(hidden_states)
93 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
94 + key_cache, value_cache = kv_cache
95 +
96 + context_layer = self.attn(
97 + position_ids,
98 + q,
99 + k,
100 + v,
101 + key_cache,
102 + value_cache,
103 + input_metadata,
104 + cache_event,
105 + )
106 +
107 + attn_output, _ = self.dense(context_layer)
108 +
109 + return attn_output
110 +
111 +
112 +class GLMMLP(nn.Module):
113 + """MLP.
114 +
115 + MLP will take the input with h hidden state, project it to 4*h
116 + hidden dimension, perform nonlinear transformation, and project the
117 + state back into h hidden dimension.
118 + """
119 +
120 + def __init__(self, config):
121 + super().__init__()
122 +
123 + self.add_bias = config.add_bias_linear
124 +
125 + # Project to 4h.
126 + self.dense_h_to_4h = ColumnParallelLinear(
127 + config.hidden_size,
128 + config.ffn_hidden_size * 2,
129 + bias=config.add_bias_linear,
130 + gather_output=False,
131 + )
132 +
133 + self.activation_func = SiluAndMul()
134 +
135 + # Project back to h.
136 + self.dense_4h_to_h = RowParallelLinear(
137 + config.ffn_hidden_size,
138 + config.hidden_size,
139 + bias=config.add_bias_linear,
140 + input_is_parallel=True,
141 + )
142 +
143 + def forward(self, hidden_states):
144 + # [s, b, 4hp]
145 + intermediate_parallel, _ = self.dense_h_to_4h(hidden_states)
146 + intermediate_parallel = self.activation_func(intermediate_parallel)
147 + # [s, b, h]
148 + output, _ = self.dense_4h_to_h(intermediate_parallel)
149 + return output
150 +
151 +
152 +class GLMBlock(nn.Module):
153 + """A single transformer layer.
154 +
155 + Transformer layer takes input with size [s, b, h] and returns an
156 + output of the same size.
157 + """
158 +
159 + def __init__(
160 + self,
161 + config,
162 + ):
163 + super().__init__()
164 + self.apply_residual_connection_post_layernorm = (
165 + config.apply_residual_connection_post_layernorm)
166 +
167 + self.fp32_residual_connection = config.fp32_residual_connection
168 +
169 + layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
170 + # Layernorm on the input data.
171 + self.input_layernorm = layer_norm_func(config.hidden_size,
172 + eps=config.layernorm_epsilon)
173 +
174 + # Self attention.
175 + self.self_attention = GLMAttention(config)
176 + self.hidden_dropout = config.hidden_dropout
177 +
178 + # Layernorm on the attention output
179 + self.post_attention_layernorm = layer_norm_func(
180 + config.hidden_size, eps=config.layernorm_epsilon)
181 +
182 + # MLP
183 + self.mlp = GLMMLP(config)
184 +
185 + def forward(
186 + self,
187 + hidden_states: torch.Tensor,
188 + position_ids: torch.Tensor,
189 + kv_cache: KVCache,
190 + input_metadata: InputMetadata,
191 + cache_event: Optional[torch.cuda.Event],
192 + ) -> torch.Tensor:
193 + # hidden_states: [num_tokens, h]
194 + # Layer norm at the beginning of the transformer layer.
195 + layernorm_output = self.input_layernorm(hidden_states)
196 + # Self attention.
197 + attention_output = self.self_attention(
198 + hidden_states=layernorm_output,
199 + position_ids=position_ids,
200 + kv_cache=kv_cache,
201 + input_metadata=input_metadata,
202 + cache_event=cache_event,
203 + )
204 +
205 + # Residual connection.
206 + if self.apply_residual_connection_post_layernorm:
207 + residual = layernorm_output
208 + else:
209 + residual = hidden_states
210 +
211 + layernorm_input = residual + attention_output
212 +
213 + # Layer norm post the self attention.
214 + layernorm_output = self.post_attention_layernorm(layernorm_input)
215 +
216 + # Second residual connection.
217 + if self.apply_residual_connection_post_layernorm:
218 + residual = layernorm_output
219 + else:
220 + residual = layernorm_input
221 +
222 + output = self.mlp(layernorm_output) + residual
223 +
224 + return output
225 +
226 +
227 +class GLMTransformer(nn.Module):
228 + """Transformer class."""
229 +
230 + def __init__(self, config):
231 + super().__init__()
232 + self.post_layer_norm = config.post_layer_norm
233 +
234 + # Number of layers.
235 + self.num_layers = config.num_layers
236 +
237 + # Transformer layers.
238 + self.layers = nn.ModuleList(
239 + [GLMBlock(config) for i in range(self.num_layers)])
240 +
241 + if self.post_layer_norm:
242 + layer_norm_func = RMSNorm if config.rmsnorm else LayerNorm
243 + # Final layer norm before output.
244 + self.final_layernorm = layer_norm_func(
245 + config.hidden_size, eps=config.layernorm_epsilon)
246 +
247 + def forward(
248 + self,
249 + hidden_states: torch.Tensor,
250 + position_ids: torch.Tensor,
251 + kv_caches: List[KVCache],
252 + input_metadata: InputMetadata,
253 + cache_events: Optional[List[torch.cuda.Event]],
254 + ) -> torch.Tensor:
255 + for i in range(self.num_layers):
256 + if cache_events is None:
257 + cache_event = None
258 + else:
259 + cache_event = cache_events[i]
260 + layer = self.layers[i]
261 + hidden_states = layer(
262 + hidden_states=hidden_states,
263 + position_ids=position_ids,
264 + kv_cache=kv_caches[i],
265 + input_metadata=input_metadata,
266 + cache_event=cache_event,
267 + )
268 + # Final layer norm.
269 + if self.post_layer_norm:
270 + hidden_states = self.final_layernorm(hidden_states)
271 +
272 + return hidden_states
273 +
274 +
275 +class ChatGLMModel(nn.Module):
276 +
277 + def __init__(self, config):
278 + super().__init__()
279 +
280 + self.embedding = VocabParallelEmbedding(config.padded_vocab_size,
281 + config.hidden_size)
282 +
283 + self.num_layers = config.num_layers
284 + self.multi_query_group_num = config.multi_query_group_num
285 + self.kv_channels = config.kv_channels
286 + self.encoder = GLMTransformer(config)
287 +
288 + self.output_layer = ColumnParallelLinear(
289 + config.hidden_size,
290 + config.padded_vocab_size,
291 + bias=False,
292 + gather_output=False,
293 + params_dtype=config.torch_dtype,
294 + )
295 +
296 + def forward(
297 + self,
298 + input_ids: torch.Tensor,
299 + position_ids: torch.Tensor,
300 + kv_caches: List[KVCache],
301 + input_metadata: InputMetadata,
302 + cache_events: Optional[List[torch.cuda.Event]],
303 + ):
304 + inputs_embeds = self.embedding(input_ids)
305 +
306 + # Run encoder.
307 + hidden_states = self.encoder(
308 + hidden_states=inputs_embeds,
309 + position_ids=position_ids,
310 + kv_caches=kv_caches,
311 + input_metadata=input_metadata,
312 + cache_events=cache_events,
313 + )
314 +
315 + return hidden_states
316 +
317 +
318 +class ChatGLMForCausalLM(nn.Module):
319 +
320 + def __init__(self, config: ChatGLMConfig):
321 + super().__init__()
322 + self.config: ChatGLMConfig = config
323 + self.transformer = ChatGLMModel(config)
324 + self.lm_head_weight = self.transformer.output_layer.weight
325 + self.sampler = Sampler(config.padded_vocab_size)
326 +
327 + def forward(
328 + self,
329 + input_ids: torch.Tensor,
330 + positions: torch.Tensor,
331 + kv_caches: List[KVCache],
332 + input_metadata: InputMetadata,
333 + cache_events: Optional[List[torch.cuda.Event]],
334 + ) -> Dict[int, SequenceOutputs]:
335 + hidden_states = self.transformer(input_ids, positions, kv_caches,
336 + input_metadata, cache_events)
337 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
338 + input_metadata)
339 + return next_tokens
340 +
341 + _column_parallel_weights = [
342 + "output_layer.weight",
343 + "embedding.weight",
344 + ]
345 + _row_parallel_weights = ["dense_4h_to_h", "self_attention.dense"]
346 +
347 + def load_weights(
348 + self,
349 + model_name_or_path: str,
350 + cache_dir: Optional[str] = None,
351 + load_format: str = "auto",
352 + revision: Optional[str] = None,
353 + ):
354 + tp_rank = get_tensor_model_parallel_rank()
355 + tp_size = get_tensor_model_parallel_world_size()
356 +
357 + q_proj_shard_size = self.config.hidden_size // tp_size
358 + kv_proj_shard_size = (self.config.hidden_size //
359 + self.config.num_attention_heads *
360 + self.config.multi_query_group_num // tp_size)
361 +
362 + mlp_hidden_shard_size = self.config.ffn_hidden_size // tp_size
363 +
364 + state_dict = self.state_dict()
365 + for name, loaded_weight in hf_model_weights_iterator(
366 + model_name_or_path, cache_dir, load_format, revision):
367 + if "word_embeddings" in name:
368 + name = name.replace(".word_embeddings", "")
369 +
370 + if name in state_dict:
371 + param = state_dict[name]
372 + if "query_key_value" in name:
373 + q_offset = q_proj_shard_size * tp_rank
374 + k_offset = (q_proj_shard_size * tp_size +
375 + kv_proj_shard_size * tp_rank)
376 + v_offset = (q_proj_shard_size * tp_size +
377 + kv_proj_shard_size * (tp_size + tp_rank))
378 + wq = loaded_weight[q_offset:q_offset + q_proj_shard_size]
379 + wk = loaded_weight[k_offset:k_offset + kv_proj_shard_size]
380 + wv = loaded_weight[v_offset:v_offset + kv_proj_shard_size]
381 + loaded_weight = torch.cat([wq, wk, wv], dim=0)
382 + param.data.copy_(loaded_weight)
383 + continue
384 +
385 + if "dense_h_to_4h" in name:
386 + w_gate = loaded_weight[mlp_hidden_shard_size *
387 + tp_rank:mlp_hidden_shard_size *
388 + (tp_rank + 1)]
389 + w_proj = loaded_weight[mlp_hidden_shard_size *
390 + (tp_size +
391 + tp_rank):mlp_hidden_shard_size *
392 + (tp_size + tp_rank + 1)]
393 + loaded_weight = torch.cat([w_gate, w_proj], dim=0)
394 + param.data.copy_(loaded_weight)
395 + continue
396 +
397 + load_tensor_parallel_weights(
398 + param,
399 + loaded_weight,
400 + name,
401 + self._column_parallel_weights,
402 + self._row_parallel_weights,
403 + tp_rank,
404 + )
405 + elif name == "transformer.rotary_pos_emb.inv_freq":
406 + continue
407 + else:
408 + print("Warning never found tensor's name:", name)
vllm/transformers_utils/config.py
+3 -2
@@ -5,9 +5,10 @@ from transformers import AutoConfig, MptConfig, PretrainedConfig
5 from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
6
7 _CONFIG_REGISTRY = {
8 - "mpt": MptConfig,
9 - "baichuan": BaiChuanConfig,
8 "aquila": AquilaConfig,
9 + "baichuan": BaiChuanConfig,
10 + "chatglm": ChatGLMConfig,
11 + "mpt": MptConfig,
12 "qwen": QWenConfig,
13 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
14 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+4 -2
@@ -1,5 +1,6 @@
1 -from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
1 from vllm.transformers_utils.configs.aquila import AquilaConfig
2 +from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
3 +from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
4 from vllm.transformers_utils.configs.qwen import QWenConfig
5 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
6 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
@@ -8,8 +9,9 @@ from vllm.transformers_utils.configs.falcon import RWConfig
9 from vllm.transformers_utils.configs.yi import YiConfig
10
11 __all__ = [
11 - "BaiChuanConfig",
12 "AquilaConfig",
13 + "BaiChuanConfig",
14 + "ChatGLMConfig",
15 "QWenConfig",
16 "RWConfig",
17 "YiConfig",
vllm/transformers_utils/configs/chatglm.py new
+68
@@ -0,0 +1,68 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/THUDM/ChatGLM2-6B
4 +from transformers import PretrainedConfig
5 +
6 +
7 +class ChatGLMConfig(PretrainedConfig):
8 + model_type = "chatglm"
9 + attribute_map = {
10 + "num_hidden_layers": "num_layers",
11 + "n_head_kv": "multi_query_group_num",
12 + }
13 +
14 + def __init__(self,
15 + num_layers=28,
16 + padded_vocab_size=65024,
17 + hidden_size=4096,
18 + ffn_hidden_size=13696,
19 + kv_channels=128,
20 + num_attention_heads=32,
21 + seq_length=2048,
22 + hidden_dropout=0.0,
23 + attention_dropout=0.0,
24 + layernorm_epsilon=1e-5,
25 + rmsnorm=True,
26 + apply_residual_connection_post_layernorm=False,
27 + post_layer_norm=True,
28 + add_bias_linear=False,
29 + add_qkv_bias=False,
30 + interleaved_qkv=False,
31 + bias_dropout_fusion=True,
32 + multi_query_attention=False,
33 + multi_query_group_num=1,
34 + apply_query_key_layer_scaling=True,
35 + attention_softmax_in_fp32=True,
36 + fp32_residual_connection=False,
37 + quantization_bit=0,
38 + pre_seq_len=None,
39 + prefix_projection=False,
40 + **kwargs):
41 + self.num_layers = num_layers
42 + self.vocab_size = padded_vocab_size
43 + self.padded_vocab_size = padded_vocab_size
44 + self.hidden_size = hidden_size
45 + self.ffn_hidden_size = ffn_hidden_size
46 + self.kv_channels = kv_channels
47 + self.num_attention_heads = num_attention_heads
48 + self.seq_length = seq_length
49 + self.hidden_dropout = hidden_dropout
50 + self.attention_dropout = attention_dropout
51 + self.layernorm_epsilon = layernorm_epsilon
52 + self.rmsnorm = rmsnorm
53 + self.apply_residual_connection_post_layernorm = (
54 + apply_residual_connection_post_layernorm)
55 + self.post_layer_norm = post_layer_norm
56 + self.add_bias_linear = add_bias_linear
57 + self.add_qkv_bias = add_qkv_bias
58 + self.bias_dropout_fusion = bias_dropout_fusion
59 + self.multi_query_attention = multi_query_attention
60 + self.multi_query_group_num = multi_query_group_num
61 + self.apply_query_key_layer_scaling = apply_query_key_layer_scaling
62 + self.attention_softmax_in_fp32 = attention_softmax_in_fp32
63 + self.fp32_residual_connection = fp32_residual_connection
64 + self.quantization_bit = quantization_bit
65 + self.pre_seq_len = pre_seq_len
66 + self.prefix_projection = prefix_projection
67 + self.interleaved_qkv = interleaved_qkv
68 + super().__init__(**kwargs)