A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fixes the incorrect argument in the prefix-prefill test cases (#3246)
Tao He committed
Mar 16, 2024 at 11:58 UTC
3123f151387d2afa49eaf3130bcee3556f2e87d2
1 file changed
+9
-2
tests/kernels/test_prefix_prefill.py
+9
-2
@@ -18,7 +18,7 @@ CUDA_DEVICES = [
18
19
20
@pytest.mark.parametrize("num_heads", NUM_HEADS)
21
-@pytest.mark.parametrize("num_queries_per_kv", NUM_HEADS)
21
+@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
22
@pytest.mark.parametrize("head_size", HEAD_SIZES)
23
@pytest.mark.parametrize("dtype", DTYPES)
24
@pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -35,6 +35,13 @@ def test_contexted_kv_attention(
35
if torch.cuda.is_available():
36
torch.cuda.manual_seed(0)
37
torch.set_default_device(device)
38
+
39
+ # Need this, otherwise when we capture the graph the process for GPU 1 would run on both
40
+ # GPU0 and GPU1 and things would hang
41
+ #
42
+ # see also similar issue: https://github.com/Dao-AILab/flash-attention/issues/523
43
+ torch.cuda.set_device(device)
44
+
45
MAX_SEQ_LEN = 1024
46
MAX_CTX_LEN = 1024
47
BS = 10
@@ -172,5 +179,5 @@ def test_contexted_kv_attention(
179
torch.cuda.synchronize()
180
end_time = time.time()
181
print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
175
- output_ref = output_ref.squeeze(0, 2)
182
+ output_ref = output_ref.reshape(output.shape)
183
assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)