A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add Internlm2 (#2666)
Fengzhe Zhou committed
Feb 2, 2024 at 01:27 UTC
cd9e60c76c776c42431b7ae523fcfe7835546d74
4 files changed
+330
README.md
+1
@@ -73,6 +73,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
73
- GPT-J (`EleutherAI/gpt-j-6b`, `nomic-ai/gpt4all-j`, etc.)
74
- GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
75
- InternLM (`internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc.)
76
+- InternLM2 (`internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.)
77
- LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
78
- Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
79
- Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
docs/source/models/supported_models.rst
+3
@@ -47,6 +47,9 @@ Alongside each architecture, we include some popular models that use it.
47
* - :code:`InternLMForCausalLM`
48
- InternLM
49
- :code:`internlm/internlm-7b`, :code:`internlm/internlm-chat-7b`, etc.
50
+ * - :code:`InternLM2ForCausalLM`
51
+ - InternLM2
52
+ - :code:`internlm/internlm2-7b`, :code:`internlm/internlm2-chat-7b`, etc.
53
* - :code:`LlamaForCausalLM`
54
- LLaMA, LLaMA-2, Vicuna, Alpaca, Koala, Guanaco
55
- :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`young-geng/koala`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -25,6 +25,7 @@ _MODELS = {
25
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27
"InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
28
+ "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30
# For decapoda-research/llama-*
31
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
vllm/model_executor/models/internlm2.py
new
+325
@@ -0,0 +1,325 @@
1
+# -*- coding: utf-8 -*-
2
+from typing import Any, Dict, List, Optional, Tuple
3
+
4
+import torch
5
+from torch import nn
6
+from transformers import PretrainedConfig
7
+
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.activation import SiluAndMul
10
+from vllm.model_executor.layers.attention import PagedAttention
11
+from vllm.model_executor.layers.layernorm import RMSNorm
12
+from vllm.model_executor.layers.linear import (LinearMethodBase,
13
+ MergedColumnParallelLinear,
14
+ QKVParallelLinear,
15
+ RowParallelLinear)
16
+from vllm.model_executor.layers.rotary_embedding import get_rope
17
+from vllm.model_executor.layers.sampler import Sampler
18
+from vllm.model_executor.layers.vocab_parallel_embedding import (
19
+ VocabParallelEmbedding, ParallelLMHead)
20
+from vllm.model_executor.parallel_utils.parallel_state import (
21
+ get_tensor_model_parallel_world_size)
22
+from vllm.model_executor.sampling_metadata import SamplingMetadata
23
+from vllm.model_executor.weight_utils import (default_weight_loader,
24
+ hf_model_weights_iterator)
25
+from vllm.sequence import SamplerOutput
26
+
27
+KVCache = Tuple[torch.Tensor, torch.Tensor]
28
+
29
+
30
+class InternLM2MLP(nn.Module):
31
+
32
+ def __init__(
33
+ self,
34
+ hidden_size: int,
35
+ intermediate_size: int,
36
+ hidden_act: str,
37
+ linear_method: Optional[LinearMethodBase] = None,
38
+ ) -> None:
39
+ super().__init__()
40
+ self.gate_up_proj = MergedColumnParallelLinear(
41
+ hidden_size, [intermediate_size] * 2,
42
+ bias=False,
43
+ linear_method=linear_method)
44
+ self.w2 = RowParallelLinear(intermediate_size,
45
+ hidden_size,
46
+ bias=False,
47
+ linear_method=linear_method)
48
+ if hidden_act != "silu":
49
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
50
+ "Only silu is supported for now.")
51
+ self.act_fn = SiluAndMul()
52
+
53
+ def forward(self, x):
54
+ gate_up, _ = self.gate_up_proj(x)
55
+ x = self.act_fn(gate_up)
56
+ x, _ = self.w2(x)
57
+ return x
58
+
59
+
60
+class InternLM2Attention(nn.Module):
61
+
62
+ def __init__(
63
+ self,
64
+ hidden_size: int,
65
+ num_heads: int,
66
+ num_kv_heads: int,
67
+ rope_theta: float = 10000,
68
+ rope_scaling: Optional[Dict[str, Any]] = None,
69
+ max_position_embeddings: int = 8192,
70
+ linear_method: Optional[LinearMethodBase] = None,
71
+ ) -> None:
72
+ super().__init__()
73
+ self.hidden_size = hidden_size
74
+ tp_size = get_tensor_model_parallel_world_size()
75
+ self.total_num_heads = num_heads
76
+ assert self.total_num_heads % tp_size == 0
77
+ self.num_heads = self.total_num_heads // tp_size
78
+ self.total_num_kv_heads = num_kv_heads
79
+ if self.total_num_kv_heads >= tp_size:
80
+ # Number of KV heads is greater than TP size, so we partition
81
+ # the KV heads across multiple tensor parallel GPUs.
82
+ assert self.total_num_kv_heads % tp_size == 0
83
+ else:
84
+ # Number of KV heads is less than TP size, so we replicate
85
+ # the KV heads across multiple tensor parallel GPUs.
86
+ assert tp_size % self.total_num_kv_heads == 0
87
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
88
+ self.head_dim = hidden_size // self.total_num_heads
89
+ self.q_size = self.num_heads * self.head_dim
90
+ self.kv_size = self.num_kv_heads * self.head_dim
91
+ self.scaling = self.head_dim**-0.5
92
+ self.rope_theta = rope_theta
93
+ self.max_position_embeddings = max_position_embeddings
94
+
95
+ self.wqkv = QKVParallelLinear(
96
+ hidden_size,
97
+ self.head_dim,
98
+ self.total_num_heads,
99
+ self.total_num_kv_heads,
100
+ bias=False,
101
+ linear_method=linear_method,
102
+ )
103
+ self.wo = RowParallelLinear(
104
+ self.total_num_heads * self.head_dim,
105
+ hidden_size,
106
+ bias=False,
107
+ linear_method=linear_method,
108
+ )
109
+
110
+ self.rotary_emb = get_rope(
111
+ self.head_dim,
112
+ rotary_dim=self.head_dim,
113
+ max_position=max_position_embeddings,
114
+ base=rope_theta,
115
+ rope_scaling=rope_scaling,
116
+ )
117
+ self.attn = PagedAttention(self.num_heads,
118
+ self.head_dim,
119
+ self.scaling,
120
+ num_kv_heads=self.num_kv_heads)
121
+
122
+ def forward(
123
+ self,
124
+ positions: torch.Tensor,
125
+ hidden_states: torch.Tensor,
126
+ kv_cache: KVCache,
127
+ input_metadata: InputMetadata,
128
+ ) -> torch.Tensor:
129
+ qkv, _ = self.wqkv(hidden_states)
130
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
131
+ q, k = self.rotary_emb(positions, q, k)
132
+ k_cache, v_cache = kv_cache
133
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134
+ output, _ = self.wo(attn_output)
135
+ return output
136
+
137
+
138
+class InternLMDecoderLayer(nn.Module):
139
+
140
+ def __init__(
141
+ self,
142
+ config: PretrainedConfig,
143
+ linear_method: Optional[LinearMethodBase] = None,
144
+ ) -> None:
145
+ super().__init__()
146
+ self.hidden_size = config.hidden_size
147
+ rope_theta = getattr(config, "rope_theta", 10000)
148
+ rope_scaling = getattr(config, "rope_scaling", None)
149
+ max_position_embeddings = getattr(config, "max_position_embeddings",
150
+ 8192)
151
+ self.attention = InternLM2Attention(
152
+ hidden_size=self.hidden_size,
153
+ num_heads=config.num_attention_heads,
154
+ num_kv_heads=config.num_key_value_heads,
155
+ rope_theta=rope_theta,
156
+ rope_scaling=rope_scaling,
157
+ max_position_embeddings=max_position_embeddings,
158
+ linear_method=linear_method,
159
+ )
160
+ self.feed_forward = InternLM2MLP(
161
+ hidden_size=self.hidden_size,
162
+ intermediate_size=config.intermediate_size,
163
+ hidden_act=config.hidden_act,
164
+ linear_method=linear_method,
165
+ )
166
+ self.attention_norm = RMSNorm(config.hidden_size,
167
+ eps=config.rms_norm_eps)
168
+ self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
169
+
170
+ def forward(
171
+ self,
172
+ positions: torch.Tensor,
173
+ hidden_states: torch.Tensor,
174
+ kv_cache: KVCache,
175
+ input_metadata: InputMetadata,
176
+ residual: Optional[torch.Tensor],
177
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
178
+ # Self Attention
179
+ if residual is None:
180
+ residual = hidden_states
181
+ hidden_states = self.attention_norm(hidden_states)
182
+ else:
183
+ hidden_states, residual = self.attention_norm(
184
+ hidden_states, residual)
185
+ hidden_states = self.attention(
186
+ positions=positions,
187
+ hidden_states=hidden_states,
188
+ kv_cache=kv_cache,
189
+ input_metadata=input_metadata,
190
+ )
191
+
192
+ # Fully Connected
193
+ hidden_states, residual = self.ffn_norm(hidden_states, residual)
194
+ hidden_states = self.feed_forward(hidden_states)
195
+ return hidden_states, residual
196
+
197
+
198
+class InternLM2Model(nn.Module):
199
+
200
+ def __init__(
201
+ self,
202
+ config: PretrainedConfig,
203
+ linear_method: Optional[LinearMethodBase] = None,
204
+ ) -> None:
205
+ super().__init__()
206
+ self.config = config
207
+ self.padding_idx = config.pad_token_id
208
+ self.vocab_size = config.vocab_size
209
+ self.tok_embeddings = VocabParallelEmbedding(
210
+ config.vocab_size,
211
+ config.hidden_size,
212
+ )
213
+ self.layers = nn.ModuleList([
214
+ InternLMDecoderLayer(config, linear_method)
215
+ for _ in range(config.num_hidden_layers)
216
+ ])
217
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
218
+
219
+ def forward(
220
+ self,
221
+ input_ids: torch.Tensor,
222
+ positions: torch.Tensor,
223
+ kv_caches: List[KVCache],
224
+ input_metadata: InputMetadata,
225
+ ) -> torch.Tensor:
226
+ hidden_states = self.tok_embeddings(input_ids)
227
+ residual = None
228
+ for i in range(len(self.layers)):
229
+ layer = self.layers[i]
230
+ hidden_states, residual = layer(
231
+ positions,
232
+ hidden_states,
233
+ kv_caches[i],
234
+ input_metadata,
235
+ residual,
236
+ )
237
+ hidden_states, _ = self.norm(hidden_states, residual)
238
+ return hidden_states
239
+
240
+
241
+class InternLM2ForCausalLM(nn.Module):
242
+
243
+ def __init__(
244
+ self,
245
+ config: PretrainedConfig,
246
+ linear_method: Optional[LinearMethodBase] = None,
247
+ ) -> None:
248
+ super().__init__()
249
+ self.config = config
250
+ self.linear_method = linear_method
251
+ self.model = InternLM2Model(config, linear_method)
252
+ self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253
+ self.sampler = Sampler(config.vocab_size)
254
+
255
+ def forward(
256
+ self,
257
+ input_ids: torch.Tensor,
258
+ positions: torch.Tensor,
259
+ kv_caches: List[KVCache],
260
+ input_metadata: InputMetadata,
261
+ ) -> torch.Tensor:
262
+ hidden_states = self.model(input_ids, positions, kv_caches,
263
+ input_metadata)
264
+ return hidden_states
265
+
266
+ def sample(
267
+ self,
268
+ hidden_states: torch.Tensor,
269
+ sampling_metadata: SamplingMetadata,
270
+ ) -> Optional[SamplerOutput]:
271
+ next_tokens = self.sampler(self.output.weight, hidden_states,
272
+ sampling_metadata)
273
+ return next_tokens
274
+
275
+ def load_weights(self,
276
+ model_name_or_path: str,
277
+ cache_dir: Optional[str] = None,
278
+ load_format: str = "auto",
279
+ revision: Optional[str] = None):
280
+ stacked_params_mapping = [
281
+ # (param_name, shard_name, shard_id)
282
+ ("gate_up_proj", "w1", 0),
283
+ ("gate_up_proj", "w3", 1),
284
+ ]
285
+ params_dict = dict(self.named_parameters())
286
+ for name, loaded_weight in hf_model_weights_iterator(
287
+ model_name_or_path, cache_dir, load_format, revision):
288
+ if "rotary_emb.inv_freq" in name:
289
+ continue
290
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
291
+ if weight_name not in name:
292
+ continue
293
+ name = name.replace(weight_name, param_name)
294
+ # Skip loading extra bias for GPTQ models.
295
+ if name.endswith(".bias") and name not in params_dict:
296
+ continue
297
+ param = params_dict[name]
298
+ weight_loader = param.weight_loader
299
+ weight_loader(param, loaded_weight, shard_id)
300
+ break
301
+ else:
302
+ # Skip loading extra bias for GPTQ models.
303
+ if name.endswith(".bias") and name not in params_dict:
304
+ continue
305
+ param = params_dict[name]
306
+ if "wqkv" in name:
307
+ config = self.config
308
+ kv_groups = config.num_attention_heads // config.num_key_value_heads
309
+ head_dim = config.hidden_size // config.num_attention_heads
310
+ loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
311
+ head_dim,
312
+ loaded_weight.shape[-1])
313
+ wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
314
+ dim=1)
315
+ wq = wq.reshape(-1, wq.shape[-1])
316
+ wk = wk.reshape(-1, wk.shape[-1])
317
+ wv = wv.reshape(-1, wv.shape[-1])
318
+ weight_loader = param.weight_loader
319
+ weight_loader(param, wq, 'q')
320
+ weight_loader(param, wk, 'k')
321
+ weight_loader(param, wv, 'v')
322
+ else:
323
+ weight_loader = getattr(param, "weight_loader",
324
+ default_weight_loader)
325
+ weight_loader(param, loaded_weight)