@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4c7755ca8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

fix biachuan-7b tp (#598)

Co-authored-by: wq.chu <wq.chu@tianrang-inc.com>

Qing committed Aug 2, 2023 at 06:41 UTC d4c7755ca88aefe749346f510be1b919186be0b6
1 file changed +37 -10
vllm/model_executor/models/baichuan.py
+37 -10
@@ -251,8 +251,8 @@ class BaiChuanForCausalLM(nn.Module):
251 return next_tokens
252
253 _column_parallel_weights = [
254 - "embed_tokens.weight", "lm_head.weight", "W_pack.weight",
255 - "gate_proj.weight", "up_proj.weight"
254 + "embed_tokens.weight",
255 + "lm_head.weight",
256 ]
257 _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
258
@@ -260,7 +260,8 @@ class BaiChuanForCausalLM(nn.Module):
260 model_name_or_path: str,
261 cache_dir: Optional[str] = None,
262 use_np_cache: bool = False):
263 - tensor_model_parallel_rank = get_tensor_model_parallel_rank()
263 + tp_world_size = get_tensor_model_parallel_world_size()
264 + tp_rank = get_tensor_model_parallel_rank()
265 state_dict = self.state_dict()
266
267 for name, loaded_weight in hf_model_weights_iterator(
@@ -268,15 +269,37 @@ class BaiChuanForCausalLM(nn.Module):
269 if "rotary_emb.inv_freq" in name:
270 continue
271
272 + if "embed_tokens" in name or "lm_head" in name:
273 + # Consider padding in the vocab size.
274 + param = state_dict[name]
275 + padded_vocab_size = param.shape[0] * tp_world_size
276 + num_extra_rows = padded_vocab_size - self.config.vocab_size
277 + extra_rows = torch.empty(num_extra_rows,
278 + loaded_weight.shape[1])
279 + extra_rows = extra_rows.to(loaded_weight)
280 + loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
281 +
282 + if "W_pack" in name:
283 + total_num_heads = self.config.num_attention_heads
284 + hidden_size = self.config.hidden_size
285 + head_size = hidden_size // total_num_heads
286 + num_heads = total_num_heads // tp_world_size
287 + head_start = tp_rank * num_heads
288 + head_end = (tp_rank + 1) * num_heads
289 +
290 + loaded_weight = loaded_weight.view(3, total_num_heads,
291 + head_size, hidden_size)
292 + loaded_weight = loaded_weight[:, head_start:head_end, :, :]
293 + loaded_weight = loaded_weight.reshape(-1, hidden_size)
294 +
295 is_gate_up_weight = False
296 for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
297 if weight_name not in name:
298 continue
299 param = state_dict[name.replace(weight_name, "gate_up_proj")]
300 shard_size = param.shape[0] // 2
277 - loaded_weight = loaded_weight[
278 - shard_size * tensor_model_parallel_rank:shard_size *
279 - (tensor_model_parallel_rank + 1)]
301 + loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
302 + (tp_rank + 1)]
303 param_slice = param.data[shard_size * stride_id:shard_size *
304 (stride_id + 1)]
305 assert param_slice.shape == loaded_weight.shape
@@ -287,7 +310,11 @@ class BaiChuanForCausalLM(nn.Module):
310 continue
311
312 param = state_dict[name]
290 - load_tensor_parallel_weights(param, loaded_weight, name,
291 - self._column_parallel_weights,
292 - self._row_parallel_weights,
293 - tensor_model_parallel_rank)
313 + load_tensor_parallel_weights(
314 + param,
315 + loaded_weight,
316 + name,
317 + self._column_parallel_weights,
318 + self._row_parallel_weights,
319 + tp_rank,
320 + )