A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Fix GPTBigcoder for distributed execution (#503)
Zhuohan Li committed
Jul 24, 2023 at 18:36 UTC
7d5a155e4aa258b0db1649bdcbf29477bf490ac2
1 file changed
+62
-24
vllm/model_executor/models/gpt_bigcode.py
+62
-24
@@ -54,15 +54,30 @@ class GPTBigCodeAttention(nn.Module):
54
assert total_num_heads % tensor_model_parallel_world_size == 0
55
self.num_heads = total_num_heads // tensor_model_parallel_world_size
56
self.head_dim = self.hidden_size // total_num_heads
57
- self.num_kv_heads = 1 if config.multi_query else self.num_heads
58
- self.kv_dim = self.num_kv_heads * self.head_dim
57
self.scale = self.head_dim**-0.5
58
61
- self.c_attn = ColumnParallelLinear(self.hidden_size,
62
- self.hidden_size + 2 * self.kv_dim,
63
- bias=True,
64
- gather_output=False,
65
- perform_initialization=False)
59
+ self.multi_query = config.multi_query
60
+ if self.multi_query:
61
+ self.num_kv_heads = 1
62
+ self.kv_dim = self.head_dim
63
+ self.c_attn_q = ColumnParallelLinear(self.hidden_size,
64
+ self.hidden_size,
65
+ bias=True,
66
+ gather_output=False,
67
+ perform_initialization=False)
68
+ self.c_attn_kv = nn.Linear(self.hidden_size,
69
+ 2 * self.kv_dim,
70
+ bias=True)
71
+ else:
72
+ self.num_kv_heads = self.num_heads
73
+ self.kv_dim = self.num_kv_heads * self.head_dim
74
+ self.c_attn = ColumnParallelLinear(self.hidden_size,
75
+ self.hidden_size +
76
+ 2 * self.kv_dim,
77
+ bias=True,
78
+ gather_output=False,
79
+ perform_initialization=False)
80
+
81
self.c_proj = RowParallelLinear(self.hidden_size,
82
self.hidden_size,
83
bias=True,
@@ -80,9 +95,14 @@ class GPTBigCodeAttention(nn.Module):
95
input_metadata: InputMetadata,
96
cache_event: Optional[torch.cuda.Event],
97
) -> torch.Tensor:
83
- qkv, _ = self.c_attn(hidden_states)
84
- q, k, v = qkv.split([self.hidden_size, self.kv_dim, self.kv_dim],
85
- dim=-1)
98
+ if self.multi_query:
99
+ q, _ = self.c_attn_q(hidden_states)
100
+ kv = self.c_attn_kv(hidden_states)
101
+ k, v = kv.split([self.kv_dim, self.kv_dim], dim=-1)
102
+ else:
103
+ qkv, _ = self.c_attn(hidden_states)
104
+ q, k, v = qkv.split([self.hidden_size, self.kv_dim, self.kv_dim],
105
+ dim=-1)
106
key_cache, value_cache = kv_cache
107
attn_output = self.attn(q, k, v, key_cache, value_cache,
108
input_metadata, cache_event)
@@ -251,21 +271,9 @@ class GPTBigCodeForCausalLM(nn.Module):
271
# NOTE: "c_attn.bias" should not be skipped.
272
continue
273
254
- param = state_dict[name]
255
-
274
if not name.startswith("transformer."):
275
name = "transformer." + name
276
259
- if name == "transformer.wte.weight":
260
- # Consider padding in the vocab size.
261
- padded_vocab_size = param.shape[
262
- 0] * tensor_model_parallel_world_size
263
- num_extra_rows = padded_vocab_size - self.config.vocab_size
264
- extra_rows = torch.empty(num_extra_rows,
265
- loaded_weight.shape[1])
266
- extra_rows = extra_rows.to(loaded_weight)
267
- loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
268
-
277
# For the fused QKV linear layer, manually shard the weights.
278
if "c_attn" in name:
279
# GPT-2's fused QKV has the shape of
@@ -291,9 +299,39 @@ class GPTBigCodeForCausalLM(nn.Module):
299
# Split the heads when using normal multi-head attention
300
wk = wk[head_size * head_start:head_size * head_end]
301
wv = wv[head_size * head_start:head_size * head_end]
294
- # Else, keep the weights as is for multi-query attention
302
+ loaded_weight = torch.cat([wq, wk, wv], dim=0)
303
+ else:
304
+ # For multi-query attention, we split the query
305
+ # but replicate the key and value.
306
+ loaded_weight_q = wq
307
+ loaded_weight_kv = torch.cat([wk, wv], dim=0)
308
+ q_weight_name = name.replace("c_attn", "c_attn_q")
309
+ kv_weight_name = name.replace("c_attn", "c_attn_kv")
310
+ load_tensor_parallel_weights(state_dict[q_weight_name],
311
+ loaded_weight_q,
312
+ q_weight_name,
313
+ self._column_parallel_weights,
314
+ self._row_parallel_weights,
315
+ tensor_model_parallel_rank)
316
+ load_tensor_parallel_weights(state_dict[kv_weight_name],
317
+ loaded_weight_kv,
318
+ kv_weight_name,
319
+ self._column_parallel_weights,
320
+ self._row_parallel_weights,
321
+ tensor_model_parallel_rank)
322
+ continue
323
+
324
+ param = state_dict[name]
325
296
- loaded_weight = torch.cat([wq, wk, wv], dim=0)
326
+ if name == "transformer.wte.weight":
327
+ # Consider padding in the vocab size.
328
+ padded_vocab_size = param.shape[
329
+ 0] * tensor_model_parallel_world_size
330
+ num_extra_rows = padded_vocab_size - self.config.vocab_size
331
+ extra_rows = torch.empty(num_extra_rows,
332
+ loaded_weight.shape[1])
333
+ extra_rows = extra_rows.to(loaded_weight)
334
+ loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
335
336
load_tensor_parallel_weights(param, loaded_weight, name,
337
self._column_parallel_weights,