A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Migrate InternLMForCausalLM to LlamaForCausalLM (#2860)
Co-authored-by: Roy <jasonailu87@gmail.com>
Philipp Moritz committed
Feb 13, 2024 at 17:12 UTC
7eacffd9512c29bfcce0963b5a19da0cd66cc22f
3 files changed
+5
-302
vllm/model_executor/models/__init__.py
+1
-1
@@ -24,7 +24,7 @@ _MODELS = {
24
"GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
25
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27
- "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
27
+ "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
28
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30
# For decapoda-research/llama-*
vllm/model_executor/models/internlm.py
deleted
-299
@@ -1,299 +0,0 @@
1
-# -*- coding: utf-8 -*-
2
-from typing import Any, Dict, List, Optional, Tuple
3
-
4
-import torch
5
-from torch import nn
6
-from transformers import LlamaConfig
7
-
8
-from vllm.model_executor.input_metadata import InputMetadata
9
-from vllm.model_executor.layers.activation import SiluAndMul
10
-from vllm.model_executor.layers.attention import PagedAttention
11
-from vllm.model_executor.layers.layernorm import RMSNorm
12
-from vllm.model_executor.layers.linear import (LinearMethodBase,
13
- MergedColumnParallelLinear,
14
- QKVParallelLinear,
15
- RowParallelLinear)
16
-from vllm.model_executor.layers.rotary_embedding import get_rope
17
-from vllm.model_executor.layers.sampler import Sampler
18
-from vllm.model_executor.layers.vocab_parallel_embedding import (
19
- VocabParallelEmbedding, ParallelLMHead)
20
-from vllm.model_executor.parallel_utils.parallel_state import (
21
- get_tensor_model_parallel_world_size)
22
-from vllm.model_executor.sampling_metadata import SamplingMetadata
23
-from vllm.model_executor.weight_utils import (default_weight_loader,
24
- hf_model_weights_iterator)
25
-from vllm.sequence import SamplerOutput
26
-
27
-KVCache = Tuple[torch.Tensor, torch.Tensor]
28
-
29
-
30
-class InternLMMLP(nn.Module):
31
-
32
- def __init__(
33
- self,
34
- hidden_size: int,
35
- intermediate_size: int,
36
- hidden_act: str,
37
- linear_method: Optional[LinearMethodBase] = None,
38
- ):
39
- super().__init__()
40
- self.gate_up_proj = MergedColumnParallelLinear(
41
- hidden_size, [intermediate_size] * 2,
42
- bias=False,
43
- linear_method=linear_method)
44
- self.down_proj = RowParallelLinear(intermediate_size,
45
- hidden_size,
46
- bias=False,
47
- linear_method=linear_method)
48
- if hidden_act != "silu":
49
- raise ValueError(f"Unsupported activation: {hidden_act}. "
50
- "Only silu is supported for now.")
51
- self.act_fn = SiluAndMul()
52
-
53
- def forward(self, x):
54
- gate_up, _ = self.gate_up_proj(x)
55
- x = self.act_fn(gate_up)
56
- x, _ = self.down_proj(x)
57
- return x
58
-
59
-
60
-class InternLMAttention(nn.Module):
61
-
62
- def __init__(
63
- self,
64
- hidden_size: int,
65
- num_heads: int,
66
- bias: bool,
67
- rope_theta: float = 10000,
68
- max_position_embeddings: int = 8192,
69
- linear_method: Optional[LinearMethodBase] = None,
70
- rope_scaling: Optional[Dict[str, Any]] = None,
71
- ):
72
- super().__init__()
73
- self.hidden_size = hidden_size
74
- tensor_model_parallel_world_size = (
75
- get_tensor_model_parallel_world_size())
76
- self.total_num_heads = num_heads
77
- assert self.total_num_heads % tensor_model_parallel_world_size == 0
78
- self.num_heads = (self.total_num_heads //
79
- tensor_model_parallel_world_size)
80
- self.head_dim = hidden_size // self.total_num_heads
81
- self.scaling = self.head_dim**-0.5
82
- self.rope_theta = rope_theta
83
- self.max_position_embeddings = max_position_embeddings
84
-
85
- self.qkv_proj = QKVParallelLinear(
86
- hidden_size,
87
- self.head_dim,
88
- self.total_num_heads,
89
- bias=bias,
90
- linear_method=linear_method,
91
- )
92
- self.o_proj = RowParallelLinear(
93
- self.total_num_heads * self.head_dim,
94
- hidden_size,
95
- bias=bias,
96
- linear_method=linear_method,
97
- )
98
- self.rotary_emb = get_rope(
99
- self.head_dim,
100
- rotary_dim=self.head_dim,
101
- max_position=self.max_position_embeddings,
102
- base=self.rope_theta,
103
- rope_scaling=rope_scaling,
104
- )
105
- self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106
-
107
- def forward(
108
- self,
109
- positions: torch.Tensor,
110
- hidden_states: torch.Tensor,
111
- kv_cache: KVCache,
112
- input_metadata: InputMetadata,
113
- ) -> torch.Tensor:
114
- qkv, _ = self.qkv_proj(hidden_states)
115
- q, k, v = qkv.chunk(chunks=3, dim=-1)
116
- q, k = self.rotary_emb(positions, q, k)
117
- k_cache, v_cache = kv_cache
118
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119
- output, _ = self.o_proj(attn_output)
120
- return output
121
-
122
-
123
-class InternLMDecoderLayer(nn.Module):
124
-
125
- def __init__(
126
- self,
127
- config: LlamaConfig,
128
- linear_method: Optional[LinearMethodBase] = None,
129
- ):
130
- super().__init__()
131
- self.hidden_size = config.hidden_size
132
- rope_theta = getattr(config, "rope_theta", 10000)
133
- max_position_embeddings = getattr(config, "max_position_embeddings",
134
- 8192)
135
- self.self_attn = InternLMAttention(
136
- hidden_size=self.hidden_size,
137
- num_heads=config.num_attention_heads,
138
- bias=config.bias,
139
- rope_theta=rope_theta,
140
- max_position_embeddings=max_position_embeddings,
141
- linear_method=linear_method,
142
- rope_scaling=getattr(config, "rope_scaling", None),
143
- )
144
- self.mlp = InternLMMLP(
145
- hidden_size=self.hidden_size,
146
- intermediate_size=config.intermediate_size,
147
- hidden_act=config.hidden_act,
148
- linear_method=linear_method,
149
- )
150
- self.input_layernorm = RMSNorm(config.hidden_size,
151
- eps=config.rms_norm_eps)
152
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
153
- eps=config.rms_norm_eps)
154
-
155
- def forward(
156
- self,
157
- positions: torch.Tensor,
158
- hidden_states: torch.Tensor,
159
- kv_cache: KVCache,
160
- input_metadata: InputMetadata,
161
- residual: Optional[torch.Tensor],
162
- ) -> Tuple[torch.Tensor, torch.Tensor]:
163
- # Self Attention
164
- if residual is None:
165
- residual = hidden_states
166
- hidden_states = self.input_layernorm(hidden_states)
167
- else:
168
- hidden_states, residual = self.input_layernorm(
169
- hidden_states, residual)
170
- hidden_states = self.self_attn(
171
- positions=positions,
172
- hidden_states=hidden_states,
173
- kv_cache=kv_cache,
174
- input_metadata=input_metadata,
175
- )
176
-
177
- # Fully Connected
178
- hidden_states, residual = self.post_attention_layernorm(
179
- hidden_states, residual)
180
- hidden_states = self.mlp(hidden_states)
181
- return hidden_states, residual
182
-
183
-
184
-class InternLMModel(nn.Module):
185
-
186
- def __init__(
187
- self,
188
- config: LlamaConfig,
189
- linear_method: Optional[LinearMethodBase] = None,
190
- ):
191
- super().__init__()
192
- self.config = config
193
- self.padding_idx = config.pad_token_id
194
- self.vocab_size = config.vocab_size
195
-
196
- vocab_size = ((config.vocab_size + 63) // 64) * 64
197
- self.embed_tokens = VocabParallelEmbedding(
198
- vocab_size,
199
- config.hidden_size,
200
- )
201
- self.layers = nn.ModuleList([
202
- InternLMDecoderLayer(config, linear_method)
203
- for _ in range(config.num_hidden_layers)
204
- ])
205
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206
-
207
- def forward(
208
- self,
209
- input_ids: torch.Tensor,
210
- positions: torch.Tensor,
211
- kv_caches: List[KVCache],
212
- input_metadata: InputMetadata,
213
- ) -> torch.Tensor:
214
- hidden_states = self.embed_tokens(input_ids)
215
- residual = None
216
- for i in range(len(self.layers)):
217
- layer = self.layers[i]
218
- hidden_states, residual = layer(
219
- positions,
220
- hidden_states,
221
- kv_caches[i],
222
- input_metadata,
223
- residual,
224
- )
225
- hidden_states, _ = self.norm(hidden_states, residual)
226
- return hidden_states
227
-
228
-
229
-class InternLMForCausalLM(nn.Module):
230
-
231
- def __init__(
232
- self,
233
- config,
234
- linear_method: Optional[LinearMethodBase] = None,
235
- ):
236
- super().__init__()
237
- self.config = config
238
- self.linear_method = linear_method
239
- self.model = InternLMModel(config, linear_method)
240
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241
- self.sampler = Sampler(config.vocab_size)
242
-
243
- def forward(
244
- self,
245
- input_ids: torch.Tensor,
246
- positions: torch.Tensor,
247
- kv_caches: List[KVCache],
248
- input_metadata: InputMetadata,
249
- ) -> torch.Tensor:
250
- hidden_states = self.model(input_ids, positions, kv_caches,
251
- input_metadata)
252
- return hidden_states
253
-
254
- def sample(
255
- self,
256
- hidden_states: torch.Tensor,
257
- sampling_metadata: SamplingMetadata,
258
- ) -> Optional[SamplerOutput]:
259
- next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260
- sampling_metadata)
261
- return next_tokens
262
-
263
- def load_weights(self,
264
- model_name_or_path: str,
265
- cache_dir: Optional[str] = None,
266
- load_format: str = "auto",
267
- revision: Optional[str] = None):
268
- stacked_params_mapping = [
269
- # (param_name, shard_name, shard_id)
270
- ("qkv_proj", "q_proj", "q"),
271
- ("qkv_proj", "k_proj", "k"),
272
- ("qkv_proj", "v_proj", "v"),
273
- ("gate_up_proj", "gate_proj", 0),
274
- ("gate_up_proj", "up_proj", 1),
275
- ]
276
- params_dict = dict(self.named_parameters())
277
- for name, loaded_weight in hf_model_weights_iterator(
278
- model_name_or_path, cache_dir, load_format, revision):
279
- if "rotary_emb.inv_freq" in name:
280
- continue
281
- for (param_name, weight_name, shard_id) in stacked_params_mapping:
282
- if weight_name not in name:
283
- continue
284
- name = name.replace(weight_name, param_name)
285
- # Skip loading extra bias for GPTQ models.
286
- if name.endswith(".bias") and name not in params_dict:
287
- continue
288
- param = params_dict[name]
289
- weight_loader = param.weight_loader
290
- weight_loader(param, loaded_weight, shard_id)
291
- break
292
- else:
293
- # Skip loading extra bias for GPTQ models.
294
- if name.endswith(".bias") and name not in params_dict:
295
- continue
296
- param = params_dict[name]
297
- weight_loader = getattr(param, "weight_loader",
298
- default_weight_loader)
299
- weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+4
-2
@@ -91,6 +91,7 @@ class LlamaAttention(nn.Module):
91
rope_scaling: Optional[Dict[str, Any]] = None,
92
max_position_embeddings: int = 8192,
93
linear_method: Optional[LinearMethodBase] = None,
94
+ bias: bool = False,
95
) -> None:
96
super().__init__()
97
self.hidden_size = hidden_size
@@ -120,13 +121,13 @@ class LlamaAttention(nn.Module):
121
self.head_dim,
122
self.total_num_heads,
123
self.total_num_kv_heads,
123
- bias=False,
124
+ bias=bias,
125
linear_method=linear_method,
126
)
127
self.o_proj = RowParallelLinear(
128
self.total_num_heads * self.head_dim,
129
hidden_size,
129
- bias=False,
130
+ bias=bias,
131
linear_method=linear_method,
132
)
133
@@ -179,6 +180,7 @@ class LlamaDecoderLayer(nn.Module):
180
rope_scaling=rope_scaling,
181
max_position_embeddings=max_position_embeddings,
182
linear_method=linear_method,
183
+ bias=getattr(config, "bias", False),
184
)
185
self.mlp = LlamaMLP(
186
hidden_size=self.hidden_size,