A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
add internlm model (#528)
Jia Guoqing committed
Aug 9, 2023 at 07:35 UTC
735ecfff61f62b78bd1455c65e8e27dc36febce1
3 files changed
+314
-4
vllm/model_executor/model_loader.py
+1
@@ -19,6 +19,7 @@ _MODEL_REGISTRY = {
19
"GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
20
"GPTJForCausalLM": GPTJForCausalLM,
21
"GPTNeoXForCausalLM": GPTNeoXForCausalLM,
22
+ "InternLMForCausalLM": InternLMForCausalLM,
23
"LlamaForCausalLM": LlamaForCausalLM,
24
"LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
25
"MPTForCausalLM": MPTForCausalLM,
vllm/model_executor/models/__init__.py
+14
-4
@@ -6,14 +6,24 @@ from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
6
from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
7
from vllm.model_executor.models.gpt_j import GPTJForCausalLM
8
from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
9
+from vllm.model_executor.models.internlm import InternLMForCausalLM
10
from vllm.model_executor.models.llama import LlamaForCausalLM
11
from vllm.model_executor.models.mpt import MPTForCausalLM
12
from vllm.model_executor.models.opt import OPTForCausalLM
13
from vllm.model_executor.models.qwen import QWenLMHeadModel
14
15
__all__ = [
15
- "BaiChuanForCausalLM", "BaichuanForCausalLM", "BloomForCausalLM",
16
- "FalconForCausalLM", "GPT2LMHeadModel", "GPTBigCodeForCausalLM",
17
- "GPTJForCausalLM", "GPTNeoXForCausalLM", "LlamaForCausalLM",
18
- "MPTForCausalLM", "OPTForCausalLM", "QWenLMHeadModel"
16
+ "BaiChuanForCausalLM",
17
+ "BaichuanForCausalLM",
18
+ "BloomForCausalLM",
19
+ "FalconForCausalLM",
20
+ "GPT2LMHeadModel",
21
+ "GPTBigCodeForCausalLM",
22
+ "GPTJForCausalLM",
23
+ "GPTNeoXForCausalLM",
24
+ "InternLMForCausalLM",
25
+ "LlamaForCausalLM",
26
+ "MPTForCausalLM",
27
+ "OPTForCausalLM",
28
+ "QWenLMHeadModel",
29
]
vllm/model_executor/models/internlm.py
new
+299
@@ -0,0 +1,299 @@
1
+# -*- coding: utf-8 -*-
2
+from typing import Dict, List, Optional, Tuple
3
+
4
+import torch
5
+from torch import nn
6
+from transformers import LlamaConfig
7
+
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.activation import SiluAndMul
10
+from vllm.model_executor.layers.layernorm import RMSNorm
11
+from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
12
+from vllm.model_executor.layers.sampler import Sampler
13
+from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
14
+ load_tensor_parallel_weights)
15
+from vllm.model_executor.parallel_utils.parallel_state import (
16
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17
+from vllm.model_executor.parallel_utils.tensor_parallel import (
18
+ VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
19
+from vllm.sequence import SequenceOutputs
20
+
21
+KVCache = Tuple[torch.Tensor, torch.Tensor]
22
+
23
+
24
+class InternLMMLP(nn.Module):
25
+
26
+ def __init__(
27
+ self,
28
+ hidden_size: int,
29
+ intermediate_size: int,
30
+ hidden_act: str,
31
+ ):
32
+ super().__init__()
33
+ self.gate_up_proj = ColumnParallelLinear(hidden_size,
34
+ 2 * intermediate_size,
35
+ bias=True,
36
+ gather_output=False,
37
+ perform_initialization=False)
38
+ self.down_proj = RowParallelLinear(intermediate_size,
39
+ hidden_size,
40
+ bias=True,
41
+ input_is_parallel=True,
42
+ perform_initialization=False)
43
+ if hidden_act != "silu":
44
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
45
+ "Only silu is supported for now.")
46
+ self.act_fn = SiluAndMul()
47
+
48
+ def forward(self, x):
49
+ gate_up, _ = self.gate_up_proj(x)
50
+ x = self.act_fn(gate_up)
51
+ x, _ = self.down_proj(x)
52
+ return x
53
+
54
+
55
+class InternLMAttention(nn.Module):
56
+
57
+ def __init__(
58
+ self,
59
+ hidden_size: int,
60
+ num_heads: int,
61
+ ):
62
+ super().__init__()
63
+ self.hidden_size = hidden_size
64
+ tensor_model_parallel_world_size = (
65
+ get_tensor_model_parallel_world_size())
66
+ self.total_num_heads = num_heads
67
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
68
+ self.num_heads = (self.total_num_heads //
69
+ tensor_model_parallel_world_size)
70
+ self.head_dim = hidden_size // self.total_num_heads
71
+ self.scaling = self.head_dim**-0.5
72
+
73
+ self.qkv_proj = ColumnParallelLinear(
74
+ hidden_size,
75
+ 3 * self.total_num_heads * self.head_dim,
76
+ bias=True,
77
+ gather_output=False,
78
+ perform_initialization=False,
79
+ )
80
+ self.o_proj = RowParallelLinear(
81
+ self.total_num_heads * self.head_dim,
82
+ hidden_size,
83
+ bias=True,
84
+ input_is_parallel=True,
85
+ perform_initialization=False,
86
+ )
87
+ self.attn = PagedAttentionWithRoPE(self.num_heads,
88
+ self.head_dim,
89
+ self.scaling,
90
+ rotary_dim=self.head_dim)
91
+
92
+ def forward(
93
+ self,
94
+ positions: torch.Tensor,
95
+ hidden_states: torch.Tensor,
96
+ kv_cache: KVCache,
97
+ input_metadata: InputMetadata,
98
+ cache_event: Optional[torch.cuda.Event],
99
+ ) -> torch.Tensor:
100
+ qkv, _ = self.qkv_proj(hidden_states)
101
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
102
+ k_cache, v_cache = kv_cache
103
+ attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
104
+ input_metadata, cache_event)
105
+ output, _ = self.o_proj(attn_output)
106
+ return output
107
+
108
+
109
+class InternLMDecoderLayer(nn.Module):
110
+
111
+ def __init__(self, config: LlamaConfig):
112
+ super().__init__()
113
+ self.hidden_size = config.hidden_size
114
+ self.self_attn = InternLMAttention(
115
+ hidden_size=self.hidden_size,
116
+ num_heads=config.num_attention_heads,
117
+ )
118
+ self.mlp = InternLMMLP(
119
+ hidden_size=self.hidden_size,
120
+ intermediate_size=config.intermediate_size,
121
+ hidden_act=config.hidden_act,
122
+ )
123
+ self.input_layernorm = RMSNorm(config.hidden_size,
124
+ eps=config.rms_norm_eps)
125
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
126
+ eps=config.rms_norm_eps)
127
+
128
+ def forward(
129
+ self,
130
+ positions: torch.Tensor,
131
+ hidden_states: torch.Tensor,
132
+ kv_cache: KVCache,
133
+ input_metadata: InputMetadata,
134
+ cache_event: Optional[torch.cuda.Event],
135
+ ) -> torch.Tensor:
136
+ # Self Attention
137
+ residual = hidden_states
138
+ hidden_states = self.input_layernorm(hidden_states)
139
+ hidden_states = self.self_attn(
140
+ positions=positions,
141
+ hidden_states=hidden_states,
142
+ kv_cache=kv_cache,
143
+ input_metadata=input_metadata,
144
+ cache_event=cache_event,
145
+ )
146
+ hidden_states = residual + hidden_states
147
+
148
+ # Fully Connected
149
+ residual = hidden_states
150
+ hidden_states = self.post_attention_layernorm(hidden_states)
151
+ hidden_states = self.mlp(hidden_states)
152
+ hidden_states = residual + hidden_states
153
+ return hidden_states
154
+
155
+
156
+class InternLMModel(nn.Module):
157
+
158
+ def __init__(self, config: LlamaConfig):
159
+ super().__init__()
160
+ self.config = config
161
+ self.padding_idx = config.pad_token_id
162
+ self.vocab_size = config.vocab_size
163
+
164
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
165
+ self.embed_tokens = VocabParallelEmbedding(
166
+ vocab_size, config.hidden_size, perform_initialization=False)
167
+ self.layers = nn.ModuleList([
168
+ InternLMDecoderLayer(config)
169
+ for _ in range(config.num_hidden_layers)
170
+ ])
171
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
172
+
173
+ def forward(
174
+ self,
175
+ input_ids: torch.Tensor,
176
+ positions: torch.Tensor,
177
+ kv_caches: List[KVCache],
178
+ input_metadata: InputMetadata,
179
+ cache_events: Optional[List[torch.cuda.Event]],
180
+ ) -> torch.Tensor:
181
+ hidden_states = self.embed_tokens(input_ids)
182
+ for i in range(len(self.layers)):
183
+ if cache_events is None:
184
+ cache_event = None
185
+ else:
186
+ cache_event = cache_events[i]
187
+ layer = self.layers[i]
188
+ hidden_states = layer(
189
+ positions,
190
+ hidden_states,
191
+ kv_caches[i],
192
+ input_metadata,
193
+ cache_event,
194
+ )
195
+ hidden_states = self.norm(hidden_states)
196
+ return hidden_states
197
+
198
+
199
+class InternLMForCausalLM(nn.Module):
200
+
201
+ def __init__(self, config):
202
+ super().__init__()
203
+ self.config = config
204
+ self.model = InternLMModel(config)
205
+ vocab_size = ((config.vocab_size + 63) // 64) * 64
206
+ self.lm_head = ColumnParallelLinear(config.hidden_size,
207
+ vocab_size,
208
+ bias=False,
209
+ gather_output=False,
210
+ perform_initialization=False)
211
+ self.sampler = Sampler(config.vocab_size)
212
+
213
+ def forward(
214
+ self,
215
+ input_ids: torch.Tensor,
216
+ positions: torch.Tensor,
217
+ kv_caches: List[KVCache],
218
+ input_metadata: InputMetadata,
219
+ cache_events: Optional[List[torch.cuda.Event]],
220
+ ) -> Dict[int, SequenceOutputs]:
221
+ hidden_states = self.model(input_ids, positions, kv_caches,
222
+ input_metadata, cache_events)
223
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
224
+ input_metadata)
225
+ return next_tokens
226
+
227
+ _column_parallel_weights = [
228
+ "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
229
+ "gate_proj.weight", "up_proj.weight"
230
+ ]
231
+ _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
232
+
233
+ def load_weights(self,
234
+ model_name_or_path: str,
235
+ cache_dir: Optional[str] = None,
236
+ use_np_cache: bool = False):
237
+ tensor_model_parallel_world_size = (
238
+ get_tensor_model_parallel_world_size())
239
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
240
+ state_dict = self.state_dict()
241
+
242
+ for name, loaded_weight in hf_model_weights_iterator(
243
+ model_name_or_path, cache_dir, use_np_cache):
244
+ if "rotary_emb.inv_freq" in name:
245
+ continue
246
+
247
+ if "embed_tokens" in name or "lm_head" in name:
248
+ param = state_dict[name]
249
+ # Consider padding in the vocab size.
250
+ padded_vocab_size = (param.shape[0] *
251
+ tensor_model_parallel_world_size)
252
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
253
+ extra_rows = torch.empty(num_extra_rows,
254
+ loaded_weight.shape[1])
255
+ extra_rows = extra_rows.to(loaded_weight)
256
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
257
+
258
+ is_attention_weight = False
259
+ for stride_id, att_weight_name in enumerate(
260
+ ["q_proj", "k_proj", "v_proj"]):
261
+ if att_weight_name not in name:
262
+ continue
263
+ param = state_dict[name.replace(att_weight_name, "qkv_proj")]
264
+ shard_size = param.shape[0] // 3
265
+ loaded_weight = loaded_weight[
266
+ shard_size * tensor_model_parallel_rank:shard_size *
267
+ (tensor_model_parallel_rank + 1)]
268
+ param_slice = param.data[shard_size * stride_id:shard_size *
269
+ (stride_id + 1)]
270
+ assert param_slice.shape == loaded_weight.shape
271
+ param_slice.copy_(loaded_weight)
272
+ is_attention_weight = True
273
+ break
274
+ if is_attention_weight:
275
+ continue
276
+
277
+ is_gate_up_weight = False
278
+ for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
279
+ if weight_name not in name:
280
+ continue
281
+ param = state_dict[name.replace(weight_name, "gate_up_proj")]
282
+ shard_size = param.shape[0] // 2
283
+ loaded_weight = loaded_weight[
284
+ shard_size * tensor_model_parallel_rank:shard_size *
285
+ (tensor_model_parallel_rank + 1)]
286
+ param_slice = param.data[shard_size * stride_id:shard_size *
287
+ (stride_id + 1)]
288
+ assert param_slice.shape == loaded_weight.shape
289
+ param_slice.copy_(loaded_weight)
290
+ is_gate_up_weight = True
291
+ break
292
+ if is_gate_up_weight:
293
+ continue
294
+
295
+ param = state_dict[name]
296
+ load_tensor_parallel_weights(param, loaded_weight, name,
297
+ self._column_parallel_weights,
298
+ self._row_parallel_weights,
299
+ tensor_model_parallel_rank)