@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb825c1e74
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix #1474 - AssertionError:assert param_slice.shape == loaded_weight.shape (#1631)

lirui committed Nov 13, 2023 at 07:53 UTC eb825c1e7401a6d9ebb2b3d8d693df0069b80ccb
1 file changed +1 -1
vllm/model_executor/models/gpt_j.py
+1 -1
@@ -250,7 +250,7 @@ class GPTJForCausalLM(nn.Module):
250 if att_weight_name not in name:
251 continue
252 param = state_dict[name.replace(att_weight_name, "qkv_proj")]
253 - shard_size = param.shape[1]
253 + shard_size = param.shape[0] // 3
254 loaded_weight = loaded_weight[shard_size * tp_rank:shard_size *
255 (tp_rank + 1)]
256 param_slice = param.data[shard_size * stride_id:shard_size *