@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d7a1c6d614
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Fix paged attention testing. (#495)

Signed-off-by: Tao Peng <jiankeng.pt@alibaba-inc.com>

Tao Peng committed Jul 25, 2023 at 12:01 UTC d7a1c6d614756b3072df3e8b52c0998035fb453f
1 file changed +9
tests/kernels/test_attention.py
+9
@@ -164,6 +164,7 @@ def run_single_query_cached_kv_attention(
164 block_size: int,
165 num_blocks: int,
166 dtype: torch.dtype,
167 + num_kv_heads: int = None,
168 ) -> None:
169 qkv = torch.empty(num_tokens,
170 3,
@@ -202,6 +203,14 @@ def run_single_query_cached_kv_attention(
203 head_mapping = torch.arange(num_heads, dtype=torch.int32, device="cuda")
204
205 scale = float(1.0 / (head_size**0.5))
206 +
207 + num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
208 + assert num_heads % num_kv_heads == 0
209 + num_queries_per_kv = num_heads // num_kv_heads
210 + head_mapping = torch.repeat_interleave(
211 + torch.arange(num_kv_heads, dtype=torch.int32, device="cuda"),
212 + num_queries_per_kv)
213 +
214 output = torch.empty(num_tokens,
215 num_heads,
216 head_size,