@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 77af974b40
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[FIX] Support non-zero CUDA devices in custom kernels (#1959)

Jee Li committed Jan 3, 2024 at 11:09 UTC 77af974b406f0947d5eb0b55f7ac4dd4516c38e3
12 files changed +74 -30
csrc/activation_kernels.cu
+4 -1
@@ -1,5 +1,6 @@
1 -#include <torch/extension.h>
1 #include <ATen/cuda/CUDAContext.h>
2 +#include <torch/extension.h>
3 +#include <c10/cuda/CUDAGuard.h>
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
@@ -36,6 +37,7 @@ void silu_and_mul(
37
38 dim3 grid(num_tokens);
39 dim3 block(std::min(d, 1024));
40 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
41 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
42 VLLM_DISPATCH_FLOATING_TYPES(
43 input.scalar_type(),
@@ -71,6 +73,7 @@ __global__ void activation_kernel(
73 int64_t num_tokens = input.numel() / d; \
74 dim3 grid(num_tokens); \
75 dim3 block(std::min(d, 1024)); \
76 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
77 const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
78 VLLM_DISPATCH_FLOATING_TYPES( \
79 input.scalar_type(), \
csrc/attention/attention_kernels.cu
+3
@@ -21,6 +21,7 @@
21
22 #include <torch/extension.h>
23 #include <ATen/cuda/CUDAContext.h>
24 +#include <c10/cuda/CUDAGuard.h>
25
26 #include "attention_dtypes.h"
27 #include "attention_utils.cuh"
@@ -616,6 +617,7 @@ void paged_attention_v1_launcher(
617
618 dim3 grid(num_heads, num_seqs, 1);
619 dim3 block(NUM_THREADS);
620 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
621 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
622 switch (head_size) {
623 // NOTE(woosuk): To reduce the compilation time, we only compile for the
@@ -784,6 +786,7 @@ void paged_attention_v2_launcher(
786 int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
787
788 dim3 block(NUM_THREADS);
789 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
790 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
791 switch (head_size) {
792 // NOTE(woosuk): To reduce the compilation time, we only compile for the
csrc/cache_kernels.cu
+5
@@ -1,5 +1,6 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3 +#include <c10/cuda/CUDAGuard.h>
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
@@ -33,6 +34,7 @@ void swap_blocks(
34 char *dst_ptr = static_cast<char*>(dst.data_ptr());
35
36 const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
37 + const at::cuda::OptionalCUDAGuard device_guard(src_device);
38 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
39 // NOTE(woosuk): This can be slow if the number of blocks is large.
40 for (const auto& pair : block_mapping) {
@@ -127,6 +129,7 @@ void copy_blocks(
129 const int numel_per_block = key_caches[0][0].numel();
130 dim3 grid(num_layers, num_pairs);
131 dim3 block(std::min(1024, numel_per_block));
132 + const at::cuda::OptionalCUDAGuard device_guard(cache_device);
133 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
134 VLLM_DISPATCH_FLOATING_TYPES(
135 key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
@@ -207,6 +210,7 @@ void reshape_and_cache(
210
211 dim3 grid(num_tokens);
212 dim3 block(std::min(num_heads * head_size, 512));
213 + const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
214 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
215 VLLM_DISPATCH_FLOATING_TYPES(
216 key.scalar_type(),
@@ -367,6 +371,7 @@ void gather_cached_kv(
371
372 dim3 grid(num_tokens);
373 dim3 block(std::min(num_heads * head_size, 512));
374 + const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
375 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
376 VLLM_DISPATCH_FLOATING_TYPES(
377 key.scalar_type(),
csrc/layernorm_kernels.cu
+3
@@ -1,5 +1,6 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3 +#include <c10/cuda/CUDAGuard.h>
4
5 #include "dispatch_utils.h"
6 #include "reduction_utils.cuh"
@@ -76,6 +77,7 @@ void rms_norm(
77
78 dim3 grid(num_tokens);
79 dim3 block(std::min(hidden_size, 1024));
80 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
81 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
82 VLLM_DISPATCH_FLOATING_TYPES(
83 input.scalar_type(),
@@ -101,6 +103,7 @@ void fused_add_rms_norm(
103
104 dim3 grid(num_tokens);
105 dim3 block(std::min(hidden_size, 1024));
106 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
107 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
108 VLLM_DISPATCH_FLOATING_TYPES(
109 input.scalar_type(),
csrc/pos_encoding_kernels.cu
+2
@@ -1,5 +1,6 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3 +#include <c10/cuda/CUDAGuard.h>
4
5 #include "cuda_compat.h"
6 #include "dispatch_utils.h"
@@ -94,6 +95,7 @@ void rotary_embedding(
95
96 dim3 grid(num_tokens);
97 dim3 block(std::min(num_heads * rot_dim / 2, 512));
98 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
99 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
100 VLLM_DISPATCH_FLOATING_TYPES(
101 query.scalar_type(),
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+2 -1
@@ -7,6 +7,7 @@
7 // half-tensor
8 #include <c10/cuda/CUDAStream.h>
9 #include <ATen/cuda/CUDATensorMethods.cuh>
10 +#include <c10/cuda/CUDAGuard.h>
11
12 #define BLOCKWIDTH 128
13 #define BLOCKHEIGHT4 16
@@ -199,7 +200,7 @@ void squeezellm_gemm(
200 (width + BLOCKWIDTH - 1) / BLOCKWIDTH
201 );
202 dim3 threads(BLOCKWIDTH);
202 -
203 + const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
204 vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
205 #ifndef USE_ROCM
206 (half2*) vec.data<at::Half>(),
tests/kernels/conftest.py
+3 -2
@@ -12,6 +12,7 @@ def create_kv_caches(
12 head_size: int,
13 dtype: torch.dtype,
14 seed: int,
15 + device: str,
16 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
17 torch.random.manual_seed(seed)
18 torch.cuda.manual_seed(seed)
@@ -23,7 +24,7 @@ def create_kv_caches(
24 for _ in range(num_layers):
25 key_cache = torch.empty(size=key_cache_shape,
26 dtype=dtype,
26 - device='cuda')
27 + device=device)
28 key_cache.uniform_(-scale, scale)
29 key_caches.append(key_cache)
30
@@ -32,7 +33,7 @@ def create_kv_caches(
33 for _ in range(num_layers):
34 value_cache = torch.empty(size=value_cache_shape,
35 dtype=dtype,
35 - device='cuda')
36 + device=device)
37 value_cache.uniform_(-scale, scale)
38 value_caches.append(value_cache)
39 return key_caches, value_caches
tests/kernels/test_activation.py
+13 -3
@@ -7,22 +7,26 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
7 NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
8 D = [512, 4096, 5120, 13824] # Arbitrary values for testing
9 SEEDS = [0]
10 +DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
11
12
13 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
14 @pytest.mark.parametrize("d", D)
15 @pytest.mark.parametrize("dtype", DTYPES)
16 @pytest.mark.parametrize("seed", SEEDS)
17 +@pytest.mark.parametrize("device", DEVICES)
18 @torch.inference_mode()
19 def test_silu_and_mul(
20 num_tokens: int,
21 d: int,
22 dtype: torch.dtype,
23 seed: int,
24 + device: int,
25 ) -> None:
26 torch.random.manual_seed(seed)
27 torch.cuda.manual_seed(seed)
25 - x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
28 + gpu_id = f"cuda:{device}"
29 + x = torch.randn(num_tokens, 2 * d, dtype=dtype, device=gpu_id)
30 layer = SiluAndMul()
31 out = layer(x)
32 ref_out = layer._forward(x)
@@ -33,16 +37,19 @@ def test_silu_and_mul(
37 @pytest.mark.parametrize("d", D)
38 @pytest.mark.parametrize("dtype", DTYPES)
39 @pytest.mark.parametrize("seed", SEEDS)
40 +@pytest.mark.parametrize("device", DEVICES)
41 @torch.inference_mode()
42 def test_gelu_new(
43 num_tokens: int,
44 d: int,
45 dtype: torch.dtype,
46 seed: int,
47 + device: int,
48 ) -> None:
49 torch.random.manual_seed(seed)
50 torch.cuda.manual_seed(seed)
45 - x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
51 + gpu_id = f"cuda:{device}"
52 + x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
53 layer = NewGELU()
54 out = layer(x)
55 ref_out = layer._forward(x)
@@ -53,15 +60,18 @@ def test_gelu_new(
60 @pytest.mark.parametrize("d", D)
61 @pytest.mark.parametrize("dtype", DTYPES)
62 @pytest.mark.parametrize("seed", SEEDS)
63 +@pytest.mark.parametrize("device", DEVICES)
64 def test_gelu_fast(
65 num_tokens: int,
66 d: int,
67 dtype: torch.dtype,
68 seed: int,
69 + device: int,
70 ) -> None:
71 torch.random.manual_seed(seed)
72 torch.cuda.manual_seed(seed)
64 - x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
73 + gpu_id = f"cuda:{device}"
74 + x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
75 layer = FastGELU()
76 out = layer(x)
77 ref_out = layer._forward(x)
tests/kernels/test_attention.py
+15 -10
@@ -24,6 +24,7 @@ HEAD_SIZES = [64, 80, 96, 112, 128, 256]
24 BLOCK_SIZES = [16, 32]
25 USE_ALIBI = [False, True]
26 SEEDS = [0]
27 +DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
28
29
30 def ref_masked_attention(
@@ -87,7 +88,7 @@ def ref_single_query_cached_kv_attention(
88 alibi_bias = None
89 if alibi_slopes is not None:
90 # Create the ALiBi bias used in the paged attention kernel.
90 - position_ids = torch.arange(context_len, device="cuda").int()
91 + position_ids = torch.arange(context_len, device=query.device).int()
92 alibi_bias = (position_ids - context_len + 1).float()
93 alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
94 1, 1, -1)
@@ -105,6 +106,7 @@ def ref_single_query_cached_kv_attention(
106 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
107 @pytest.mark.parametrize("dtype", DTYPES)
108 @pytest.mark.parametrize("seed", SEEDS)
109 +@pytest.mark.parametrize("device", DEVICES)
110 def test_paged_attention(
111 kv_cache_factory,
112 version: str,
@@ -115,18 +117,19 @@ def test_paged_attention(
117 block_size: int,
118 dtype: torch.dtype,
119 seed: int,
120 + device: int,
121 ) -> None:
122 random.seed(seed)
123 torch.random.manual_seed(seed)
124 torch.cuda.manual_seed(seed)
122 -
125 + gpu_id = f"cuda:{device}"
126 scale = float(1.0 / (head_size**0.5))
127 num_query_heads, num_kv_heads = num_heads
128 query = torch.empty(num_seqs,
129 num_query_heads,
130 head_size,
131 dtype=dtype,
129 - device="cuda")
132 + device=gpu_id)
133 query.uniform_(-scale, scale)
134
135 assert num_query_heads % num_kv_heads == 0
@@ -135,12 +138,12 @@ def test_paged_attention(
138 if use_alibi:
139 alibi_slopes = torch.randn(num_query_heads,
140 dtype=torch.float,
138 - device="cuda")
141 + device=gpu_id)
142
143 context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
144 context_lens[-1] = MAX_SEQ_LEN
145 max_context_len = max(context_lens)
143 - context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
146 + context_lens = torch.tensor(context_lens, dtype=torch.int, device=gpu_id)
147
148 # Create the block tables.
149 max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
@@ -151,12 +154,12 @@ def test_paged_attention(
154 for _ in range(max_num_blocks_per_seq)
155 ]
156 block_tables.append(block_table)
154 - block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
157 + block_tables = torch.tensor(block_tables, dtype=torch.int, device=gpu_id)
158
159 # Create the KV caches.
160 key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
161 num_kv_heads, head_size, dtype,
159 - seed)
162 + seed, gpu_id)
163 key_cache, value_cache = key_caches[0], value_caches[0]
164
165 # Call the paged attention kernel.
@@ -249,7 +252,7 @@ def ref_multi_query_kv_attention(
252 attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
253 diagonal=1)
254 attn_mask = attn_mask * torch.finfo(dtype).min
252 - attn_mask = attn_mask.to(dtype=dtype, device="cuda")
255 + attn_mask = attn_mask.to(dtype=dtype, device=query.device)
256
257 ref_output = ref_masked_attention(
258 query[start_idx:end_idx],
@@ -269,6 +272,7 @@ def ref_multi_query_kv_attention(
272 @pytest.mark.parametrize("head_size", HEAD_SIZES)
273 @pytest.mark.parametrize("dtype", DTYPES)
274 @pytest.mark.parametrize("seed", SEEDS)
275 +@pytest.mark.parametrize("device", DEVICES)
276 @torch.inference_mode()
277 def test_multi_query_kv_attention(
278 num_seqs: int,
@@ -276,11 +280,12 @@ def test_multi_query_kv_attention(
280 head_size: int,
281 dtype: torch.dtype,
282 seed: int,
283 + device: int,
284 ) -> None:
285 random.seed(seed)
286 torch.random.manual_seed(seed)
287 torch.cuda.manual_seed(seed)
283 -
288 + gpu_id = f"cuda:{device}"
289 # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
290 # As the xformers library is already tested with its own tests, we can use
291 # a smaller MAX_SEQ_LEN here.
@@ -294,7 +299,7 @@ def test_multi_query_kv_attention(
299 num_query_heads + 2 * num_kv_heads,
300 head_size,
301 dtype=dtype,
297 - device="cuda")
302 + device=gpu_id)
303 qkv.uniform_(-scale, scale)
304 query, key, value = qkv.split(
305 [num_query_heads, num_kv_heads, num_kv_heads], dim=1)
tests/kernels/test_cache.py
+11 -6
@@ -14,6 +14,7 @@ BLOCK_SIZES = [8, 16, 32]
14 NUM_BLOCKS = [1024, 36000] # Arbitrary values for testing
15 NUM_MAPPINGS = [256] # Arbitrary values for testing
16 SEEDS = [0]
17 +DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
18
19
20 @pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@@ -24,6 +25,7 @@ SEEDS = [0]
25 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
26 @pytest.mark.parametrize("dtype", DTYPES)
27 @pytest.mark.parametrize("seed", SEEDS)
28 +@pytest.mark.parametrize("device", DEVICES)
29 @torch.inference_mode()
30 def test_copy_blocks(
31 kv_cache_factory,
@@ -35,11 +37,12 @@ def test_copy_blocks(
37 num_blocks: int,
38 dtype: torch.dtype,
39 seed: int,
40 + device: int,
41 ) -> None:
42 random.seed(seed)
43 torch.random.manual_seed(seed)
44 torch.cuda.manual_seed(seed)
42 -
45 + gpu_id = f"cuda:{device}"
46 # Generate random block mappings where each source block is mapped to two
47 # destination blocks.
48 assert 2 * num_mappings <= num_blocks
@@ -56,7 +59,7 @@ def test_copy_blocks(
59 # Create the KV caches.
60 key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
61 num_layers, num_heads,
59 - head_size, dtype, seed)
62 + head_size, dtype, seed, gpu_id)
63
64 # Clone the KV caches.
65 cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
@@ -88,6 +91,7 @@ def test_copy_blocks(
91 @pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
92 @pytest.mark.parametrize("dtype", DTYPES)
93 @pytest.mark.parametrize("seed", SEEDS)
94 +@pytest.mark.parametrize("device", DEVICES)
95 @torch.inference_mode()
96 def test_reshape_and_cache(
97 kv_cache_factory,
@@ -98,28 +102,29 @@ def test_reshape_and_cache(
102 num_blocks: int,
103 dtype: torch.dtype,
104 seed: int,
105 + device: int,
106 ) -> None:
107 random.seed(seed)
108 torch.random.manual_seed(seed)
109 torch.cuda.manual_seed(seed)
105 -
110 + gpu_id = f"cuda:{device}"
111 # Create a random slot mapping.
112 num_slots = block_size * num_blocks
113 slot_mapping = random.sample(range(num_slots), num_tokens)
109 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device="cuda")
114 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=gpu_id)
115
116 qkv = torch.randn(num_tokens,
117 3,
118 num_heads,
119 head_size,
120 dtype=dtype,
116 - device="cuda")
121 + device=gpu_id)
122 _, key, value = qkv.unbind(dim=1)
123
124 # Create the KV caches.
125 key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
126 num_heads, head_size, dtype,
122 - seed)
127 + seed, gpu_id)
128 key_cache, value_cache = key_caches[0], value_caches[0]
129
130 # Clone the KV caches.
tests/kernels/test_layernorm.py
+6 -3
@@ -8,6 +8,7 @@ NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8 HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
9 ADD_RESIDUAL = [False, True]
10 SEEDS = [0]
11 +DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
12
13
14 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -15,6 +16,7 @@ SEEDS = [0]
16 @pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
17 @pytest.mark.parametrize("dtype", DTYPES)
18 @pytest.mark.parametrize("seed", SEEDS)
19 +@pytest.mark.parametrize("device", DEVICES)
20 @torch.inference_mode()
21 def test_rms_norm(
22 num_tokens: int,
@@ -22,14 +24,15 @@ def test_rms_norm(
24 add_residual: bool,
25 dtype: torch.dtype,
26 seed: int,
27 + device: int,
28 ) -> None:
29 torch.random.manual_seed(seed)
30 torch.cuda.manual_seed(seed)
28 -
29 - layer = RMSNorm(hidden_size).to(dtype).cuda()
31 + gpu_id = f"cuda:{device}"
32 + layer = RMSNorm(hidden_size).to(dtype=dtype, device=gpu_id)
33 layer.weight.data.normal_(mean=1.0, std=0.1)
34 scale = 1 / (2 * hidden_size)
32 - x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda")
35 + x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=gpu_id)
36 x *= scale
37 residual = torch.randn_like(x) * scale if add_residual else None
38
tests/kernels/test_pos_encoding.py
+7 -4
@@ -13,6 +13,7 @@ NUM_HEADS = [7, 17] # Arbitrary values for testing
13 BATCH_SIZES = [1, 5] # Arbitrary values for testing
14 SEQ_LENS = [11, 8192] # Arbitrary values for testing
15 SEEDS = [0]
16 +DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
17
18
19 @pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -23,6 +24,7 @@ SEEDS = [0]
24 @pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
25 @pytest.mark.parametrize("dtype", DTYPES)
26 @pytest.mark.parametrize("seed", SEEDS)
27 +@pytest.mark.parametrize("device", DEVICES)
28 @torch.inference_mode()
29 def test_rotary_embedding(
30 is_neox_style: bool,
@@ -33,6 +35,7 @@ def test_rotary_embedding(
35 rotary_dim: Optional[int],
36 dtype: torch.dtype,
37 seed: int,
38 + device: int,
39 max_position: int = 8192,
40 base: int = 10000,
41 ) -> None:
@@ -40,20 +43,20 @@ def test_rotary_embedding(
43 rotary_dim = head_size
44 torch.random.manual_seed(seed)
45 torch.cuda.manual_seed(seed)
43 -
46 + gpu_id = f"cuda:{device}"
47 if rotary_dim is None:
48 rotary_dim = head_size
49 rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
47 - rope = rope.to(dtype).cuda()
50 + rope = rope.to(dtype=dtype, device=gpu_id)
51
52 positions = torch.randint(0,
53 max_position, (batch_size, seq_len),
51 - device="cuda")
54 + device=gpu_id)
55 query = torch.randn(batch_size,
56 seq_len,
57 num_heads * head_size,
58 dtype=dtype,
56 - device="cuda")
59 + device=gpu_id)
60 key = torch.randn_like(query)
61
62 # NOTE(woosuk): The reference implementation should be executed first