A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Fix paged attention testing. (#495)
Signed-off-by: Tao Peng <jiankeng.pt@alibaba-inc.com>
Tao Peng committed
Jul 25, 2023 at 12:01 UTC
d7a1c6d614756b3072df3e8b52c0998035fb453f
1 file changed
+9
tests/kernels/test_attention.py
+9
@@ -164,6 +164,7 @@ def run_single_query_cached_kv_attention(
164
block_size: int,
165
num_blocks: int,
166
dtype: torch.dtype,
167
+ num_kv_heads: int = None,
168
) -> None:
169
qkv = torch.empty(num_tokens,
170
3,
@@ -202,6 +203,14 @@ def run_single_query_cached_kv_attention(
203
head_mapping = torch.arange(num_heads, dtype=torch.int32, device="cuda")
204
205
scale = float(1.0 / (head_size**0.5))
206
+
207
+ num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
208
+ assert num_heads % num_kv_heads == 0
209
+ num_queries_per_kv = num_heads // num_kv_heads
210
+ head_mapping = torch.repeat_interleave(
211
+ torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
212
+ num_queries_per_kv)
213
+
214
output = torch.empty(num_tokens,
215
num_heads,
216
head_size,