A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Address Phi modeling update 2 (#2428)
Gary Hui committed
Jan 13, 2024 at 04:16 UTC
7878958c0de9da8c372495dab7d4f25257894c4f
2 files changed
+60
-62
vllm/model_executor/models/__init__.py
+1
-1
@@ -33,7 +33,7 @@ _MODELS = {
33
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
34
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
35
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
36
- "PhiForCausalLM": ("phi_1_5", "PhiForCausalLM"),
36
+ "PhiForCausalLM": ("phi", "PhiForCausalLM"),
37
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
38
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
39
"YiForCausalLM": ("yi", "YiForCausalLM"),
vllm/model_executor/models/phi.py
+59
-61
@@ -62,20 +62,6 @@ from vllm.sequence import SamplerOutput
62
KVCache = Tuple[torch.Tensor, torch.Tensor]
63
64
65
-class PhiEmbedding(nn.Module):
66
-
67
- def __init__(self, config: PretrainedConfig):
68
- super().__init__()
69
-
70
- self.wte = VocabParallelEmbedding(
71
- config.vocab_size,
72
- config.hidden_size,
73
- )
74
-
75
- def forward(self, input_ids: torch.LongTensor):
76
- return self.wte(input_ids)
77
-
78
-
65
class PhiAttention(nn.Module):
66
67
def __init__(self,
@@ -93,27 +79,22 @@ class PhiAttention(nn.Module):
79
tensor_model_parallel_world_size)
80
81
# pylint: disable=C0103
96
- self.Wqkv = QKVParallelLinear(
97
- self.hidden_size,
98
- self.head_size,
99
- self.total_num_heads,
100
- linear_method=linear_method,
101
- )
82
self.qkv_proj = QKVParallelLinear(
103
- config.hidden_size,
83
+ self.hidden_size,
84
self.head_size,
85
self.total_num_heads,
106
- bias=False,
86
+ bias=True,
87
linear_method=linear_method,
88
)
109
- self.out_proj = RowParallelLinear(
89
+ self.dense = RowParallelLinear(
90
self.hidden_size,
91
self.hidden_size,
92
linear_method=linear_method,
93
)
94
95
scaling = self.head_size**-0.5
116
- rotary_dim = config.rotary_dim
96
+ rotary_dim = int(config.partial_rotary_factor *
97
+ (config.hidden_size // config.num_attention_heads))
98
assert rotary_dim % 2 == 0
99
100
# pylint: disable=C0301
@@ -136,12 +117,12 @@ class PhiAttention(nn.Module):
117
kv_cache: KVCache,
118
input_metadata: InputMetadata,
119
) -> torch.Tensor:
139
- qkv, _ = self.Wqkv(hidden_states)
120
+ qkv, _ = self.qkv_proj(hidden_states)
121
q, k, v = qkv.chunk(chunks=3, dim=-1)
122
q, k = self.rotary_emb(position_ids, q, k)
123
k_cache, v_cache = kv_cache
124
attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
144
- output, _ = self.out_proj(attn_output)
125
+ output, _ = self.dense(attn_output)
126
return output
127
128
@@ -166,8 +147,7 @@ class PhiMLP(nn.Module):
147
linear_method=linear_method,
148
)
149
quant_config = getattr(linear_method, "quant_config", None)
169
- self.act = get_act_fn(config.activation_function, quant_config,
170
- n_inner)
150
+ self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
151
152
def forward(self, hidden_states):
153
hidden_states, _ = self.fc1(hidden_states)
@@ -182,9 +162,9 @@ class PhiLayer(nn.Module):
162
config: PretrainedConfig,
163
linear_method: Optional[LinearMethodBase] = None):
164
super().__init__()
185
- self.ln = nn.LayerNorm(config.hidden_size,
186
- eps=config.layer_norm_epsilon)
187
- self.mixer = PhiAttention(config, linear_method)
165
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
166
+ eps=config.layer_norm_eps)
167
+ self.self_attn = PhiAttention(config, linear_method)
168
self.mlp = PhiMLP(config, linear_method)
169
170
def forward(
@@ -195,8 +175,8 @@ class PhiLayer(nn.Module):
175
input_metadata: InputMetadata,
176
) -> torch.Tensor:
177
residual = hidden_states
198
- hidden_states = self.ln(hidden_states)
199
- attn_outputs = self.mixer(
178
+ hidden_states = self.input_layernorm(hidden_states)
179
+ attn_outputs = self.self_attn(
180
position_ids=position_ids,
181
hidden_states=hidden_states,
182
kv_cache=kv_cache,
@@ -215,11 +195,14 @@ class PhiModel(nn.Module):
195
super().__init__()
196
self.config = config
197
self.linear_method = linear_method
218
- self.embd = PhiEmbedding(config)
219
- self.h = nn.ModuleList([
198
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
199
+ config.hidden_size)
200
+ self.layers = nn.ModuleList([
201
PhiLayer(config, linear_method)
202
for _ in range(config.num_hidden_layers)
203
])
204
+ self.final_layernorm = nn.LayerNorm(config.hidden_size,
205
+ eps=config.layer_norm_eps)
206
207
def forward(
208
self,
@@ -228,27 +211,19 @@ class PhiModel(nn.Module):
211
kv_caches: List[KVCache],
212
input_metadata: InputMetadata,
213
) -> torch.Tensor:
231
- hidden_states = self.embd(input_ids)
214
+ hidden_states = self.embed_tokens(input_ids)
215
for i in range(self.config.num_hidden_layers):
233
- layer = self.h[i]
216
+ layer = self.layers[i]
217
hidden_states = layer(
218
positions,
219
hidden_states,
220
kv_caches[i],
221
input_metadata,
222
)
240
- return hidden_states
241
-
223
243
-class PhiCausalLMHead(nn.Module):
224
+ hidden_states = self.final_layernorm(hidden_states)
225
245
- def __init__(self, config: PretrainedConfig):
246
- super().__init__()
247
- self.ln = nn.LayerNorm(config.hidden_size,
248
- eps=config.layer_norm_epsilon)
249
- self.linear = ParallelLMHead(config.vocab_size,
250
- config.hidden_size,
251
- bias=True)
226
+ return hidden_states
227
228
229
class PhiForCausalLM(nn.Module):
@@ -260,8 +235,11 @@ class PhiForCausalLM(nn.Module):
235
self.config = config
236
self.linear_method = linear_method
237
263
- self.transformer = PhiModel(config, linear_method)
264
- self.lm_head = PhiCausalLMHead(config)
238
+ self.model = PhiModel(config, linear_method)
239
+
240
+ self.lm_head = ParallelLMHead(config.vocab_size,
241
+ config.hidden_size,
242
+ bias=True)
243
self.sampler = Sampler(config.vocab_size)
244
245
def forward(
@@ -271,9 +249,9 @@ class PhiForCausalLM(nn.Module):
249
kv_caches: List[KVCache],
250
input_metadata: InputMetadata,
251
) -> torch.Tensor:
274
- hidden_states = self.transformer(input_ids, positions, kv_caches,
275
- input_metadata)
276
- hidden_states = self.lm_head.ln(hidden_states)
252
+ hidden_states = self.model(input_ids, positions, kv_caches,
253
+ input_metadata)
254
+
255
return hidden_states
256
257
def sample(
@@ -281,7 +259,7 @@ class PhiForCausalLM(nn.Module):
259
hidden_states: torch.Tensor,
260
sampling_metadata: SamplingMetadata,
261
) -> Optional[SamplerOutput]:
284
- head = self.lm_head.linear
262
+ head = self.lm_head
263
next_tokens = self.sampler(head.weight, hidden_states,
264
sampling_metadata, head.bias)
265
return next_tokens
@@ -291,17 +269,37 @@ class PhiForCausalLM(nn.Module):
269
cache_dir: Optional[str] = None,
270
load_format: str = "auto",
271
revision: Optional[str] = None):
272
+ stacked_params_mapping = [
273
+ # (param_name, shard_name, shard_id)
274
+ ("qkv_proj", "q_proj", "q"),
275
+ ("qkv_proj", "k_proj", "k"),
276
+ ("qkv_proj", "v_proj", "v")
277
+ ]
278
params_dict = dict(self.named_parameters())
279
+
280
for name, loaded_weight in hf_model_weights_iterator(
281
model_name_or_path, cache_dir, load_format, revision):
282
if "rotary_emb.inv_freq" in name:
283
continue
284
300
- # Skip loading extra bias for GPTQ models.
301
- if name.endswith(".bias") and name not in params_dict:
302
- continue
303
- # pylint: disable=E1136
304
- param = params_dict[name]
305
- weight_loader = getattr(param, "weight_loader",
306
- default_weight_loader)
307
- weight_loader(param, loaded_weight)
285
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
286
+ if weight_name not in name:
287
+ continue
288
+ name = name.replace(weight_name, param_name)
289
+ # Skip loading extra bias for GPTQ models.
290
+ if name.endswith(".bias") and name not in params_dict:
291
+ continue
292
+ param = params_dict[name]
293
+ weight_loader = param.weight_loader
294
+ weight_loader(param, loaded_weight, shard_id)
295
+ break
296
+ else:
297
+ # Skip loading extra bias for GPTQ models.
298
+ if name.endswith(".bias") and name not in params_dict:
299
+ continue
300
+ # pylint: disable=E1136
301
+
302
+ param = params_dict[name]
303
+ weight_loader = getattr(param, "weight_loader",
304
+ default_weight_loader)
305
+ weight_loader(param, loaded_weight)