A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Weight loading for GPTBigCode (#313)
Zhuohan Li committed
Jun 29, 2023 at 22:14 UTC
598dc4b79a0078fe14e0c134d0b53b4842a8b227
2 files changed
+18
-4
vllm/model_executor/models/gpt_bigcode.py
+15
-3
@@ -235,17 +235,28 @@ class GPTBigCodeForCausalLM(nn.Module):
235
continue
236
237
param = state_dict[name]
238
-
238
+
239
+ if not name.startswith("transformer."):
240
+ name = "transformer." + name
241
+
242
+ if name == "transformer.wte.weight":
243
+ # Consider padding in the vocab size.
244
+ padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
245
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
246
+ extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
247
+ extra_rows = extra_rows.to(loaded_weight)
248
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
249
+
250
def _expand_mqa_mha(qkv_array, n_head, head_dim):
251
"""manipulates along axis=0 from MQA to MHA
252
inputs: qkv_array.shape=((n_heads + 2) * head_dim, hidden_dim)
253
with n_heads for q, then 1 for k, 1 for 1 v, times head dim
254
return: qkv_array.shape=(3 * n_heads * head_dim, hidden_dim)
244
-
255
+
256
TODO: this function is no longer needed once vllm supports MQA.
257
"""
258
qkv_array = qkv_array.numpy()
248
-
259
+
260
dims_q = n_head * head_dim
261
q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim), axis=0)
262
# q is fine, but k & v have not replicated shape along the first axis
@@ -285,6 +296,7 @@ class GPTBigCodeForCausalLM(nn.Module):
296
loaded_weight = loaded_weight.reshape(-1)
297
else:
298
raise ValueError(f"Unexpected parameter name {name}")
299
+
300
load_tensor_parallel_weights(param, loaded_weight, name,
301
self._column_parallel_weights,
302
self._row_parallel_weights,
vllm/model_executor/weight_utils.py
+3
-1
@@ -98,7 +98,9 @@ def load_tensor_parallel_weights(
98
shard_size * tensor_model_parallel_rank
99
:shard_size * (tensor_model_parallel_rank + 1)]
100
break
101
- assert param.shape == loaded_weight.shape
101
+ assert param.shape == loaded_weight.shape, (
102
+ f"{param_name} shape mismatch between model and checkpoint: "
103
+ f"{param.shape} != {loaded_weight.shape}")
104
param.data.copy_(loaded_weight)
105
106