@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f8584756d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Fix mqa is false case in gpt_bigcode (#806)

zhaoyang-star committed Aug 22, 2023 at 13:22 UTC 4f8584756d79234bd8207201273490907340b4bb
1 file changed +11 -8
vllm/model_executor/models/gpt_bigcode.py
+11 -8
@@ -49,10 +49,11 @@ class GPTBigCodeAttention(nn.Module):
49 super().__init__()
50 self.hidden_size = config.hidden_size
51 total_num_heads = config.num_attention_heads
52 - tensor_model_parallel_world_size = (
52 + self.tensor_model_parallel_world_size = (
53 get_tensor_model_parallel_world_size())
54 - assert total_num_heads % tensor_model_parallel_world_size == 0
55 - self.num_heads = total_num_heads // tensor_model_parallel_world_size
54 + assert total_num_heads % self.tensor_model_parallel_world_size == 0
55 + self.num_heads = (total_num_heads //
56 + self.tensor_model_parallel_world_size)
57 self.head_dim = self.hidden_size // total_num_heads
58 self.scale = self.head_dim**-0.5
59
@@ -101,7 +102,10 @@ class GPTBigCodeAttention(nn.Module):
102 k, v = kv.split([self.kv_dim, self.kv_dim], dim=-1)
103 else:
104 qkv, _ = self.c_attn(hidden_states)
104 - q, k, v = qkv.split([self.hidden_size, self.kv_dim, self.kv_dim],
105 + q, k, v = qkv.split([
106 + self.hidden_size // self.tensor_model_parallel_world_size,
107 + self.kv_dim, self.kv_dim
108 + ],
109 dim=-1)
110 key_cache, value_cache = kv_cache
111 attn_output = self.attn(q, k, v, key_cache, value_cache,
@@ -255,8 +259,6 @@ class GPTBigCodeForCausalLM(nn.Module):
259 model_name_or_path: str,
260 cache_dir: Optional[str] = None,
261 use_np_cache: bool = False):
258 - tensor_model_parallel_world_size = (
259 - get_tensor_model_parallel_world_size())
262 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
263 state_dict = self.state_dict()
264
@@ -286,7 +288,8 @@ class GPTBigCodeForCausalLM(nn.Module):
288 hidden_size = self.config.hidden_size
289 head_size = hidden_size // total_num_heads
290 total_kv_size = head_size * total_num_kv_heads
289 - num_heads = total_num_heads // tensor_model_parallel_world_size
291 + num_heads = (total_num_heads //
292 + self.tensor_model_parallel_world_size)
293 head_start = tensor_model_parallel_rank * num_heads
294 head_end = (tensor_model_parallel_rank + 1) * num_heads
295
@@ -326,7 +329,7 @@ class GPTBigCodeForCausalLM(nn.Module):
329 if name == "transformer.wte.weight":
330 # Consider padding in the vocab size.
331 padded_vocab_size = param.shape[
329 - 0] * tensor_model_parallel_world_size
332 + 0] * self.tensor_model_parallel_world_size
333 num_extra_rows = padded_vocab_size - self.config.vocab_size
334 extra_rows = torch.empty(num_extra_rows,
335 loaded_weight.shape[1])