A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Fix mqa is false case in gpt_bigcode (#806)
zhaoyang-star committed
Aug 22, 2023 at 13:22 UTC
4f8584756d79234bd8207201273490907340b4bb
1 file changed
+11
-8
vllm/model_executor/models/gpt_bigcode.py
+11
-8
@@ -49,10 +49,11 @@ class GPTBigCodeAttention(nn.Module):
49
super().__init__()
50
self.hidden_size = config.hidden_size
51
total_num_heads = config.num_attention_heads
52
- tensor_model_parallel_world_size = (
52
+ self.tensor_model_parallel_world_size = (
53
get_tensor_model_parallel_world_size())
54
- assert total_num_heads % tensor_model_parallel_world_size == 0
55
- self.num_heads = total_num_heads // tensor_model_parallel_world_size
54
+ assert total_num_heads % self.tensor_model_parallel_world_size == 0
55
+ self.num_heads = (total_num_heads //
56
+ self.tensor_model_parallel_world_size)
57
self.head_dim = self.hidden_size // total_num_heads
58
self.scale = self.head_dim**-0.5
59
@@ -101,7 +102,10 @@ class GPTBigCodeAttention(nn.Module):
102
k, v = kv.split([self.kv_dim, self.kv_dim], dim=-1)
103
else:
104
qkv, _ = self.c_attn(hidden_states)
104
- q, k, v = qkv.split([self.hidden_size, self.kv_dim, self.kv_dim],
105
+ q, k, v = qkv.split([
106
+ self.hidden_size // self.tensor_model_parallel_world_size,
107
+ self.kv_dim, self.kv_dim
108
+ ],
109
dim=-1)
110
key_cache, value_cache = kv_cache
111
attn_output = self.attn(q, k, v, key_cache, value_cache,
@@ -255,8 +259,6 @@ class GPTBigCodeForCausalLM(nn.Module):
259
model_name_or_path: str,
260
cache_dir: Optional[str] = None,
261
use_np_cache: bool = False):
258
- tensor_model_parallel_world_size = (
259
- get_tensor_model_parallel_world_size())
262
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
263
state_dict = self.state_dict()
264
@@ -286,7 +288,8 @@ class GPTBigCodeForCausalLM(nn.Module):
288
hidden_size = self.config.hidden_size
289
head_size = hidden_size // total_num_heads
290
total_kv_size = head_size * total_num_kv_heads
289
- num_heads = total_num_heads // tensor_model_parallel_world_size
291
+ num_heads = (total_num_heads //
292
+ self.tensor_model_parallel_world_size)
293
head_start = tensor_model_parallel_rank * num_heads
294
head_end = (tensor_model_parallel_rank + 1) * num_heads
295
@@ -326,7 +329,7 @@ class GPTBigCodeForCausalLM(nn.Module):
329
if name == "transformer.wte.weight":
330
# Consider padding in the vocab size.
331
padded_vocab_size = param.shape[
329
- 0] * tensor_model_parallel_world_size
332
+ 0] * self.tensor_model_parallel_world_size
333
num_extra_rows = padded_vocab_size - self.config.vocab_size
334
extra_rows = torch.empty(num_extra_rows,
335
loaded_weight.shape[1])