@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6428f1d051
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Support MPT with GQA (#1938)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Megha Agarwal committed Dec 12, 2023 at 10:16 UTC 6428f1d051d731a9eca192950591e7a5a2788cb2
2 files changed +28 -6
vllm/model_executor/layers/attention.py
+8 -4
@@ -138,7 +138,8 @@ class PagedAttention(nn.Module):
138 input_metadata.attn_bias = attn_bias
139 else:
140 input_metadata.attn_bias = _make_alibi_bias(
141 - self.alibi_slopes, batch_size, seq_len, query.dtype)
141 + self.alibi_slopes, self.num_kv_heads, batch_size,
142 + seq_len, query.dtype)
143
144 # TODO(woosuk): Too many view operations. Let's try to reduce them
145 # in the future for code readability.
@@ -180,31 +181,34 @@ class PagedAttention(nn.Module):
181
182 def _make_alibi_bias(
183 alibi_slopes: torch.Tensor,
184 + num_kv_heads: int,
185 batch_size: int,
186 seq_len: int,
187 dtype: torch.dtype,
188 ) -> LowerTriangularMaskWithTensorBias:
187 - bias = torch.arange(seq_len, dtype=dtype)
189 + bias = torch.arange(seq_len, dtype=dtype, device="cuda")
190 # NOTE(zhuohan): HF uses
191 # `bias = bias[None, :].repeat(prompt_len, 1)`
192 # here. We find that both biases give the same results, but
193 # the bias below more accurately follows the original ALiBi
194 # paper.
195 bias = bias[None, :] - bias[:, None]
194 - bias = bias.to(alibi_slopes.device)
196
197 # When using custom attention bias, xformers requires the bias to
198 # be sliced from a tensor whose length is a multiple of 8.
199 padded_len = (seq_len + 7) // 8 * 8
200 + num_heads = alibi_slopes.shape[0]
201 bias = torch.empty(
202 batch_size,
201 - alibi_slopes.shape[0],
203 + num_heads,
204 seq_len,
205 padded_len,
206 device=alibi_slopes.device,
207 dtype=dtype,
208 )[:, :, :, :seq_len].copy_(bias)
209 bias.mul_(alibi_slopes[:, None, None])
210 + if num_heads != num_kv_heads:
211 + bias = bias.unflatten(1, (num_kv_heads, num_heads // num_kv_heads))
212 attn_bias = LowerTriangularMaskWithTensorBias(bias)
213 return attn_bias
214
vllm/model_executor/models/mpt.py
+20 -2
@@ -50,9 +50,14 @@ class MPTAttention(nn.Module):
50 super().__init__()
51 self.d_model = config.d_model
52 self.total_num_heads = config.n_heads
53 + self.head_dim = self.d_model // self.total_num_heads
54 self.clip_qkv = config.attn_config["clip_qkv"]
55 self.qk_ln = config.attn_config["qk_ln"]
56 self.alibi_bias_max = config.attn_config["alibi_bias_max"]
57 + if "kv_n_heads" in config.attn_config:
58 + self.total_num_kv_heads = config.attn_config['kv_n_heads']
59 + else:
60 + self.total_num_kv_heads = self.total_num_heads
61 assert not config.attn_config["prefix_lm"]
62 assert config.attn_config["alibi"]
63
@@ -61,6 +66,7 @@ class MPTAttention(nn.Module):
66 self.d_model,
67 self.d_model // self.total_num_heads,
68 self.total_num_heads,
69 + self.total_num_kv_heads,
70 bias=not config.no_bias,
71 linear_method=linear_method,
72 )
@@ -78,6 +84,17 @@ class MPTAttention(nn.Module):
84 assert self.total_num_heads % tp_world_size == 0
85 self.num_heads = self.total_num_heads // tp_world_size
86
87 + if self.total_num_kv_heads >= tp_world_size:
88 + # Number of KV heads is greater than TP size, so we partition
89 + # the KV heads across multiple tensor parallel GPUs.
90 + assert self.total_num_kv_heads % tp_world_size == 0
91 + else:
92 + # Number of KV heads is less than TP size, so we replicate
93 + # the KV heads across multiple tensor parallel GPUs.
94 + assert tp_world_size % self.total_num_kv_heads == 0
95 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_world_size)
96 + self.q_size = self.num_heads * self.head_dim
97 + self.kv_size = self.num_kv_heads * self.head_dim
98 # Create the alibi slopes and slice them.
99 tp_rank = get_tensor_model_parallel_rank()
100 head_start = tp_rank * self.num_heads
@@ -91,7 +108,8 @@ class MPTAttention(nn.Module):
108 self.attn = PagedAttention(self.num_heads,
109 self.head_dim,
110 scaling,
94 - alibi_slopes=alibi_slopes)
111 + alibi_slopes=alibi_slopes,
112 + num_kv_heads=self.num_kv_heads)
113
114 def forward(
115 self,
@@ -105,7 +123,7 @@ class MPTAttention(nn.Module):
123 qkv, _ = self.Wqkv(hidden_states)
124 if self.clip_qkv is not None:
125 qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
108 - q, k, v = qkv.chunk(chunks=3, dim=-1)
126 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
127 if self.qk_ln:
128 q = self.q_ln(q)
129 k = self.k_ln(k)