@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3123f15138
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fixes the incorrect argument in the prefix-prefill test cases (#3246)

Tao He committed Mar 16, 2024 at 11:58 UTC 3123f151387d2afa49eaf3130bcee3556f2e87d2
1 file changed +9 -2
tests/kernels/test_prefix_prefill.py
+9 -2
@@ -18,7 +18,7 @@ CUDA_DEVICES = [
18
19
20 @pytest.mark.parametrize("num_heads", NUM_HEADS)
21 -@pytest.mark.parametrize("num_queries_per_kv", NUM_HEADS)
21 +@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
22 @pytest.mark.parametrize("head_size", HEAD_SIZES)
23 @pytest.mark.parametrize("dtype", DTYPES)
24 @pytest.mark.parametrize("device", CUDA_DEVICES)
@@ -35,6 +35,13 @@ def test_contexted_kv_attention(
35 if torch.cuda.is_available():
36 torch.cuda.manual_seed(0)
37 torch.set_default_device(device)
38 +
39 + # Need this, otherwise when we capture the graph the process for GPU 1 would run on both
40 + # GPU0 and GPU1 and things would hang
41 + #
42 + # see also similar issue: https://github.com/Dao-AILab/flash-attention/issues/523
43 + torch.cuda.set_device(device)
44 +
45 MAX_SEQ_LEN = 1024
46 MAX_CTX_LEN = 1024
47 BS = 10
@@ -172,5 +179,5 @@ def test_contexted_kv_attention(
179 torch.cuda.synchronize()
180 end_time = time.time()
181 print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
175 - output_ref = output_ref.squeeze(0, 2)
182 + output_ref = output_ref.reshape(output.shape)
183 assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)