@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 598dc4b79a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Weight loading for GPTBigCode (#313)

Zhuohan Li committed Jun 29, 2023 at 22:14 UTC 598dc4b79a0078fe14e0c134d0b53b4842a8b227
2 files changed +18 -4
vllm/model_executor/models/gpt_bigcode.py
+15 -3
@@ -235,17 +235,28 @@ class GPTBigCodeForCausalLM(nn.Module):
235 continue
236
237 param = state_dict[name]
238 -
238 +
239 + if not name.startswith("transformer."):
240 + name = "transformer." + name
241 +
242 + if name == "transformer.wte.weight":
243 + # Consider padding in the vocab size.
244 + padded_vocab_size = param.shape[0] * tensor_model_parallel_world_size
245 + num_extra_rows = padded_vocab_size - self.config.vocab_size
246 + extra_rows = torch.empty(num_extra_rows, loaded_weight.shape[1])
247 + extra_rows = extra_rows.to(loaded_weight)
248 + loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
249 +
250 def _expand_mqa_mha(qkv_array, n_head, head_dim):
251 """manipulates along axis=0 from MQA to MHA
252 inputs: qkv_array.shape=((n_heads + 2) * head_dim, hidden_dim)
253 with n_heads for q, then 1 for k, 1 for 1 v, times head dim
254 return: qkv_array.shape=(3 * n_heads * head_dim, hidden_dim)
244 -
255 +
256 TODO: this function is no longer needed once vllm supports MQA.
257 """
258 qkv_array = qkv_array.numpy()
248 -
259 +
260 dims_q = n_head * head_dim
261 q, k, v = np.split(qkv_array, (dims_q, dims_q + head_dim), axis=0)
262 # q is fine, but k & v have not replicated shape along the first axis
@@ -285,6 +296,7 @@ class GPTBigCodeForCausalLM(nn.Module):
296 loaded_weight = loaded_weight.reshape(-1)
297 else:
298 raise ValueError(f"Unexpected parameter name {name}")
299 +
300 load_tensor_parallel_weights(param, loaded_weight, name,
301 self._column_parallel_weights,
302 self._row_parallel_weights,
vllm/model_executor/weight_utils.py
+3 -1
@@ -98,7 +98,9 @@ def load_tensor_parallel_weights(
98 shard_size * tensor_model_parallel_rank
99 :shard_size * (tensor_model_parallel_rank + 1)]
100 break
101 - assert param.shape == loaded_weight.shape
101 + assert param.shape == loaded_weight.shape, (
102 + f"{param_name} shape mismatch between model and checkpoint: "
103 + f"{param.shape} != {loaded_weight.shape}")
104 param.data.copy_(loaded_weight)
105
106