A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
fix biachuan-7b tp (#598)
Co-authored-by: wq.chu <wq.chu@tianrang-inc.com>
Qing committed
Aug 2, 2023 at 06:41 UTC
d4c7755ca88aefe749346f510be1b919186be0b6
1 file changed
+37
-10
vllm/model_executor/models/baichuan.py
+37
-10
@@ -251,8 +251,8 @@ class BaiChuanForCausalLM(nn.Module):
251
return next_tokens
252
253
_column_parallel_weights = [
254
- "embed_tokens.weight", "lm_head.weight", "W_pack.weight",
255
- "gate_proj.weight", "up_proj.weight"
254
+ "embed_tokens.weight",
255
+ "lm_head.weight",
256
]
257
_row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
258
@@ -260,7 +260,8 @@ class BaiChuanForCausalLM(nn.Module):
260
model_name_or_path: str,
261
cache_dir: Optional[str] = None,
262
use_np_cache: bool = False):
263
- tensor_model_parallel_rank = get_tensor_model_parallel_rank()
263
+ tp_world_size = get_tensor_model_parallel_world_size()
264
+ tp_rank = get_tensor_model_parallel_rank()
265
state_dict = self.state_dict()
266
267
for name, loaded_weight in hf_model_weights_iterator(
@@ -268,15 +269,37 @@ class BaiChuanForCausalLM(nn.Module):
269
if "rotary_emb.inv_freq" in name:
270
continue
271
272
+ if "embed_tokens" in name or "lm_head" in name:
273
+ # Consider padding in the vocab size.
274
+ param = state_dict[name]
275
+ padded_vocab_size = param.shape[0] * tp_world_size
276
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
277
+ extra_rows = torch.empty(num_extra_rows,
278
+ loaded_weight.shape[1])
279
+ extra_rows = extra_rows.to(loaded_weight)
280
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
281
+
282
+ if "W_pack" in name:
283
+ total_num_heads = self.config.num_attention_heads
284
+ hidden_size = self.config.hidden_size
285
+ head_size = hidden_size // total_num_heads
286
+ num_heads = total_num_heads // tp_world_size
287
+ head_start = tp_rank * num_heads
288
+ head_end = (tp_rank + 1) * num_heads
289
+
290
+ loaded_weight = loaded_weight.view(3, total_num_heads,
291
+ head_size, hidden_size)
292
+ loaded_weight = loaded_weight[:, head_start:head_end, :, :]
293
+ loaded_weight = loaded_weight.reshape(-1, hidden_size)
294
+
295
is_gate_up_weight = False
296
for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
297
if weight_name not in name:
298
continue
299
param = state_dict[name.replace(weight_name, "gate_up_proj")]
300
shard_size = param.shape[0] // 2
277
- loaded_weight = loaded_weight[
278
- shard_size * tensor_model_parallel_rank:shard_size *
279
- (tensor_model_parallel_rank + 1)]
301
+ loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
302
+ (tp_rank + 1)]
303
param_slice = param.data[shard_size * stride_id:shard_size *
304
(stride_id + 1)]
305
assert param_slice.shape == loaded_weight.shape
@@ -287,7 +310,11 @@ class BaiChuanForCausalLM(nn.Module):
310
continue
311
312
param = state_dict[name]
290
- load_tensor_parallel_weights(param, loaded_weight, name,
291
- self._column_parallel_weights,
292
- self._row_parallel_weights,
293
- tensor_model_parallel_rank)
313
+ load_tensor_parallel_weights(
314
+ param,
315
+ loaded_weight,
316
+ name,
317
+ self._column_parallel_weights,
318
+ self._row_parallel_weights,
319
+ tp_rank,
320
+ )