@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7878958c0d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Address Phi modeling update 2 (#2428)

Gary Hui committed Jan 13, 2024 at 04:16 UTC 7878958c0de9da8c372495dab7d4f25257894c4f
2 files changed +60 -62
vllm/model_executor/models/__init__.py
+1 -1
@@ -33,7 +33,7 @@ _MODELS = {
33 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
34 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
35 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
36 - "PhiForCausalLM": ("phi_1_5", "PhiForCausalLM"),
36 + "PhiForCausalLM": ("phi", "PhiForCausalLM"),
37 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
38 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
39 "YiForCausalLM": ("yi", "YiForCausalLM"),
vllm/model_executor/models/phi.py
+59 -61
@@ -62,20 +62,6 @@ from vllm.sequence import SamplerOutput
62 KVCache = Tuple[torch.Tensor, torch.Tensor]
63
64
65 -class PhiEmbedding(nn.Module):
66 -
67 - def __init__(self, config: PretrainedConfig):
68 - super().__init__()
69 -
70 - self.wte = VocabParallelEmbedding(
71 - config.vocab_size,
72 - config.hidden_size,
73 - )
74 -
75 - def forward(self, input_ids: torch.LongTensor):
76 - return self.wte(input_ids)
77 -
78 -
65 class PhiAttention(nn.Module):
66
67 def __init__(self,
@@ -93,27 +79,22 @@ class PhiAttention(nn.Module):
79 tensor_model_parallel_world_size)
80
81 # pylint: disable=C0103
96 - self.Wqkv = QKVParallelLinear(
97 - self.hidden_size,
98 - self.head_size,
99 - self.total_num_heads,
100 - linear_method=linear_method,
101 - )
82 self.qkv_proj = QKVParallelLinear(
103 - config.hidden_size,
83 + self.hidden_size,
84 self.head_size,
85 self.total_num_heads,
106 - bias=False,
86 + bias=True,
87 linear_method=linear_method,
88 )
109 - self.out_proj = RowParallelLinear(
89 + self.dense = RowParallelLinear(
90 self.hidden_size,
91 self.hidden_size,
92 linear_method=linear_method,
93 )
94
95 scaling = self.head_size**-0.5
116 - rotary_dim = config.rotary_dim
96 + rotary_dim = int(config.partial_rotary_factor *
97 + (config.hidden_size // config.num_attention_heads))
98 assert rotary_dim % 2 == 0
99
100 # pylint: disable=C0301
@@ -136,12 +117,12 @@ class PhiAttention(nn.Module):
117 kv_cache: KVCache,
118 input_metadata: InputMetadata,
119 ) -> torch.Tensor:
139 - qkv, _ = self.Wqkv(hidden_states)
120 + qkv, _ = self.qkv_proj(hidden_states)
121 q, k, v = qkv.chunk(chunks=3, dim=-1)
122 q, k = self.rotary_emb(position_ids, q, k)
123 k_cache, v_cache = kv_cache
124 attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
144 - output, _ = self.out_proj(attn_output)
125 + output, _ = self.dense(attn_output)
126 return output
127
128
@@ -166,8 +147,7 @@ class PhiMLP(nn.Module):
147 linear_method=linear_method,
148 )
149 quant_config = getattr(linear_method, "quant_config", None)
169 - self.act = get_act_fn(config.activation_function, quant_config,
170 - n_inner)
150 + self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
151
152 def forward(self, hidden_states):
153 hidden_states, _ = self.fc1(hidden_states)
@@ -182,9 +162,9 @@ class PhiLayer(nn.Module):
162 config: PretrainedConfig,
163 linear_method: Optional[LinearMethodBase] = None):
164 super().__init__()
185 - self.ln = nn.LayerNorm(config.hidden_size,
186 - eps=config.layer_norm_epsilon)
187 - self.mixer = PhiAttention(config, linear_method)
165 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
166 + eps=config.layer_norm_eps)
167 + self.self_attn = PhiAttention(config, linear_method)
168 self.mlp = PhiMLP(config, linear_method)
169
170 def forward(
@@ -195,8 +175,8 @@ class PhiLayer(nn.Module):
175 input_metadata: InputMetadata,
176 ) -> torch.Tensor:
177 residual = hidden_states
198 - hidden_states = self.ln(hidden_states)
199 - attn_outputs = self.mixer(
178 + hidden_states = self.input_layernorm(hidden_states)
179 + attn_outputs = self.self_attn(
180 position_ids=position_ids,
181 hidden_states=hidden_states,
182 kv_cache=kv_cache,
@@ -215,11 +195,14 @@ class PhiModel(nn.Module):
195 super().__init__()
196 self.config = config
197 self.linear_method = linear_method
218 - self.embd = PhiEmbedding(config)
219 - self.h = nn.ModuleList([
198 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
199 + config.hidden_size)
200 + self.layers = nn.ModuleList([
201 PhiLayer(config, linear_method)
202 for _ in range(config.num_hidden_layers)
203 ])
204 + self.final_layernorm = nn.LayerNorm(config.hidden_size,
205 + eps=config.layer_norm_eps)
206
207 def forward(
208 self,
@@ -228,27 +211,19 @@ class PhiModel(nn.Module):
211 kv_caches: List[KVCache],
212 input_metadata: InputMetadata,
213 ) -> torch.Tensor:
231 - hidden_states = self.embd(input_ids)
214 + hidden_states = self.embed_tokens(input_ids)
215 for i in range(self.config.num_hidden_layers):
233 - layer = self.h[i]
216 + layer = self.layers[i]
217 hidden_states = layer(
218 positions,
219 hidden_states,
220 kv_caches[i],
221 input_metadata,
222 )
240 - return hidden_states
241 -
223
243 -class PhiCausalLMHead(nn.Module):
224 + hidden_states = self.final_layernorm(hidden_states)
225
245 - def __init__(self, config: PretrainedConfig):
246 - super().__init__()
247 - self.ln = nn.LayerNorm(config.hidden_size,
248 - eps=config.layer_norm_epsilon)
249 - self.linear = ParallelLMHead(config.vocab_size,
250 - config.hidden_size,
251 - bias=True)
226 + return hidden_states
227
228
229 class PhiForCausalLM(nn.Module):
@@ -260,8 +235,11 @@ class PhiForCausalLM(nn.Module):
235 self.config = config
236 self.linear_method = linear_method
237
263 - self.transformer = PhiModel(config, linear_method)
264 - self.lm_head = PhiCausalLMHead(config)
238 + self.model = PhiModel(config, linear_method)
239 +
240 + self.lm_head = ParallelLMHead(config.vocab_size,
241 + config.hidden_size,
242 + bias=True)
243 self.sampler = Sampler(config.vocab_size)
244
245 def forward(
@@ -271,9 +249,9 @@ class PhiForCausalLM(nn.Module):
249 kv_caches: List[KVCache],
250 input_metadata: InputMetadata,
251 ) -> torch.Tensor:
274 - hidden_states = self.transformer(input_ids, positions, kv_caches,
275 - input_metadata)
276 - hidden_states = self.lm_head.ln(hidden_states)
252 + hidden_states = self.model(input_ids, positions, kv_caches,
253 + input_metadata)
254 +
255 return hidden_states
256
257 def sample(
@@ -281,7 +259,7 @@ class PhiForCausalLM(nn.Module):
259 hidden_states: torch.Tensor,
260 sampling_metadata: SamplingMetadata,
261 ) -> Optional[SamplerOutput]:
284 - head = self.lm_head.linear
262 + head = self.lm_head
263 next_tokens = self.sampler(head.weight, hidden_states,
264 sampling_metadata, head.bias)
265 return next_tokens
@@ -291,17 +269,37 @@ class PhiForCausalLM(nn.Module):
269 cache_dir: Optional[str] = None,
270 load_format: str = "auto",
271 revision: Optional[str] = None):
272 + stacked_params_mapping = [
273 + # (param_name, shard_name, shard_id)
274 + ("qkv_proj", "q_proj", "q"),
275 + ("qkv_proj", "k_proj", "k"),
276 + ("qkv_proj", "v_proj", "v")
277 + ]
278 params_dict = dict(self.named_parameters())
279 +
280 for name, loaded_weight in hf_model_weights_iterator(
281 model_name_or_path, cache_dir, load_format, revision):
282 if "rotary_emb.inv_freq" in name:
283 continue
284
300 - # Skip loading extra bias for GPTQ models.
301 - if name.endswith(".bias") and name not in params_dict:
302 - continue
303 - # pylint: disable=E1136
304 - param = params_dict[name]
305 - weight_loader = getattr(param, "weight_loader",
306 - default_weight_loader)
307 - weight_loader(param, loaded_weight)
285 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
286 + if weight_name not in name:
287 + continue
288 + name = name.replace(weight_name, param_name)
289 + # Skip loading extra bias for GPTQ models.
290 + if name.endswith(".bias") and name not in params_dict:
291 + continue
292 + param = params_dict[name]
293 + weight_loader = param.weight_loader
294 + weight_loader(param, loaded_weight, shard_id)
295 + break
296 + else:
297 + # Skip loading extra bias for GPTQ models.
298 + if name.endswith(".bias") and name not in params_dict:
299 + continue
300 + # pylint: disable=E1136
301 +
302 + param = params_dict[name]
303 + weight_loader = getattr(param, "weight_loader",
304 + default_weight_loader)
305 + weight_loader(param, loaded_weight)