A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[FIX] Support non-zero CUDA devices in custom kernels (#1959)
Jee Li committed
Jan 3, 2024 at 11:09 UTC
77af974b406f0947d5eb0b55f7ac4dd4516c38e3
12 files changed
+74
-30
csrc/activation_kernels.cu
+4
-1
@@ -1,5 +1,6 @@
1
-#include <torch/extension.h>
1
#include <ATen/cuda/CUDAContext.h>
2
+#include <torch/extension.h>
3
+#include <c10/cuda/CUDAGuard.h>
4
5
#include "cuda_compat.h"
6
#include "dispatch_utils.h"
@@ -36,6 +37,7 @@ void silu_and_mul(
37
38
dim3 grid(num_tokens);
39
dim3 block(std::min(d, 1024));
40
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
41
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
42
VLLM_DISPATCH_FLOATING_TYPES(
43
input.scalar_type(),
@@ -71,6 +73,7 @@ __global__ void activation_kernel(
73
int64_t num_tokens = input.numel() / d; \
74
dim3 grid(num_tokens); \
75
dim3 block(std::min(d, 1024)); \
76
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
77
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
78
VLLM_DISPATCH_FLOATING_TYPES( \
79
input.scalar_type(), \
csrc/attention/attention_kernels.cu
+3
@@ -21,6 +21,7 @@
21
22
#include <torch/extension.h>
23
#include <ATen/cuda/CUDAContext.h>
24
+#include <c10/cuda/CUDAGuard.h>
25
26
#include "attention_dtypes.h"
27
#include "attention_utils.cuh"
@@ -616,6 +617,7 @@ void paged_attention_v1_launcher(
617
618
dim3 grid(num_heads, num_seqs, 1);
619
dim3 block(NUM_THREADS);
620
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
621
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
622
switch (head_size) {
623
// NOTE(woosuk): To reduce the compilation time, we only compile for the
@@ -784,6 +786,7 @@ void paged_attention_v2_launcher(
786
int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
787
788
dim3 block(NUM_THREADS);
789
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
790
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
791
switch (head_size) {
792
// NOTE(woosuk): To reduce the compilation time, we only compile for the
csrc/cache_kernels.cu
+5
@@ -1,5 +1,6 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
+#include <c10/cuda/CUDAGuard.h>
4
5
#include "cuda_compat.h"
6
#include "dispatch_utils.h"
@@ -33,6 +34,7 @@ void swap_blocks(
34
char *dst_ptr = static_cast<char*>(dst.data_ptr());
35
36
const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
37
+ const at::cuda::OptionalCUDAGuard device_guard(src_device);
38
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
39
// NOTE(woosuk): This can be slow if the number of blocks is large.
40
for (const auto& pair : block_mapping) {
@@ -127,6 +129,7 @@ void copy_blocks(
129
const int numel_per_block = key_caches[0][0].numel();
130
dim3 grid(num_layers, num_pairs);
131
dim3 block(std::min(1024, numel_per_block));
132
+ const at::cuda::OptionalCUDAGuard device_guard(cache_device);
133
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
134
VLLM_DISPATCH_FLOATING_TYPES(
135
key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
@@ -207,6 +210,7 @@ void reshape_and_cache(
210
211
dim3 grid(num_tokens);
212
dim3 block(std::min(num_heads * head_size, 512));
213
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
214
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
215
VLLM_DISPATCH_FLOATING_TYPES(
216
key.scalar_type(),
@@ -367,6 +371,7 @@ void gather_cached_kv(
371
372
dim3 grid(num_tokens);
373
dim3 block(std::min(num_heads * head_size, 512));
374
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
375
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
376
VLLM_DISPATCH_FLOATING_TYPES(
377
key.scalar_type(),
csrc/layernorm_kernels.cu
+3
@@ -1,5 +1,6 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
+#include <c10/cuda/CUDAGuard.h>
4
5
#include "dispatch_utils.h"
6
#include "reduction_utils.cuh"
@@ -76,6 +77,7 @@ void rms_norm(
77
78
dim3 grid(num_tokens);
79
dim3 block(std::min(hidden_size, 1024));
80
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
81
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
82
VLLM_DISPATCH_FLOATING_TYPES(
83
input.scalar_type(),
@@ -101,6 +103,7 @@ void fused_add_rms_norm(
103
104
dim3 grid(num_tokens);
105
dim3 block(std::min(hidden_size, 1024));
106
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
107
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
108
VLLM_DISPATCH_FLOATING_TYPES(
109
input.scalar_type(),
csrc/pos_encoding_kernels.cu
+2
@@ -1,5 +1,6 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
+#include <c10/cuda/CUDAGuard.h>
4
5
#include "cuda_compat.h"
6
#include "dispatch_utils.h"
@@ -94,6 +95,7 @@ void rotary_embedding(
95
96
dim3 grid(num_tokens);
97
dim3 block(std::min(num_heads * rot_dim / 2, 512));
98
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
99
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
100
VLLM_DISPATCH_FLOATING_TYPES(
101
query.scalar_type(),
csrc/quantization/squeezellm/quant_cuda_kernel.cu
+2
-1
@@ -7,6 +7,7 @@
7
// half-tensor
8
#include <c10/cuda/CUDAStream.h>
9
#include <ATen/cuda/CUDATensorMethods.cuh>
10
+#include <c10/cuda/CUDAGuard.h>
11
12
#define BLOCKWIDTH 128
13
#define BLOCKHEIGHT4 16
@@ -199,7 +200,7 @@ void squeezellm_gemm(
200
(width + BLOCKWIDTH - 1) / BLOCKWIDTH
201
);
202
dim3 threads(BLOCKWIDTH);
202
-
203
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));
204
vllm::squeezellm::NUQ4MatMulKernel<<<blocks, threads>>>(
205
#ifndef USE_ROCM
206
(half2*) vec.data<at::Half>(),
tests/kernels/conftest.py
+3
-2
@@ -12,6 +12,7 @@ def create_kv_caches(
12
head_size: int,
13
dtype: torch.dtype,
14
seed: int,
15
+ device: str,
16
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
17
torch.random.manual_seed(seed)
18
torch.cuda.manual_seed(seed)
@@ -23,7 +24,7 @@ def create_kv_caches(
24
for _ in range(num_layers):
25
key_cache = torch.empty(size=key_cache_shape,
26
dtype=dtype,
26
- device='cuda')
27
+ device=device)
28
key_cache.uniform_(-scale, scale)
29
key_caches.append(key_cache)
30
@@ -32,7 +33,7 @@ def create_kv_caches(
33
for _ in range(num_layers):
34
value_cache = torch.empty(size=value_cache_shape,
35
dtype=dtype,
35
- device='cuda')
36
+ device=device)
37
value_cache.uniform_(-scale, scale)
38
value_caches.append(value_cache)
39
return key_caches, value_caches
tests/kernels/test_activation.py
+13
-3
@@ -7,22 +7,26 @@ DTYPES = [torch.half, torch.bfloat16, torch.float]
7
NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
8
D = [512, 4096, 5120, 13824] # Arbitrary values for testing
9
SEEDS = [0]
10
+DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
11
12
13
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
14
@pytest.mark.parametrize("d", D)
15
@pytest.mark.parametrize("dtype", DTYPES)
16
@pytest.mark.parametrize("seed", SEEDS)
17
+@pytest.mark.parametrize("device", DEVICES)
18
@torch.inference_mode()
19
def test_silu_and_mul(
20
num_tokens: int,
21
d: int,
22
dtype: torch.dtype,
23
seed: int,
24
+ device: int,
25
) -> None:
26
torch.random.manual_seed(seed)
27
torch.cuda.manual_seed(seed)
25
- x = torch.randn(num_tokens, 2 * d, dtype=dtype, device="cuda")
28
+ gpu_id = f"cuda:{device}"
29
+ x = torch.randn(num_tokens, 2 * d, dtype=dtype, device=gpu_id)
30
layer = SiluAndMul()
31
out = layer(x)
32
ref_out = layer._forward(x)
@@ -33,16 +37,19 @@ def test_silu_and_mul(
37
@pytest.mark.parametrize("d", D)
38
@pytest.mark.parametrize("dtype", DTYPES)
39
@pytest.mark.parametrize("seed", SEEDS)
40
+@pytest.mark.parametrize("device", DEVICES)
41
@torch.inference_mode()
42
def test_gelu_new(
43
num_tokens: int,
44
d: int,
45
dtype: torch.dtype,
46
seed: int,
47
+ device: int,
48
) -> None:
49
torch.random.manual_seed(seed)
50
torch.cuda.manual_seed(seed)
45
- x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
51
+ gpu_id = f"cuda:{device}"
52
+ x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
53
layer = NewGELU()
54
out = layer(x)
55
ref_out = layer._forward(x)
@@ -53,15 +60,18 @@ def test_gelu_new(
60
@pytest.mark.parametrize("d", D)
61
@pytest.mark.parametrize("dtype", DTYPES)
62
@pytest.mark.parametrize("seed", SEEDS)
63
+@pytest.mark.parametrize("device", DEVICES)
64
def test_gelu_fast(
65
num_tokens: int,
66
d: int,
67
dtype: torch.dtype,
68
seed: int,
69
+ device: int,
70
) -> None:
71
torch.random.manual_seed(seed)
72
torch.cuda.manual_seed(seed)
64
- x = torch.randn(num_tokens, d, dtype=dtype, device="cuda")
73
+ gpu_id = f"cuda:{device}"
74
+ x = torch.randn(num_tokens, d, dtype=dtype, device=gpu_id)
75
layer = FastGELU()
76
out = layer(x)
77
ref_out = layer._forward(x)
tests/kernels/test_attention.py
+15
-10
@@ -24,6 +24,7 @@ HEAD_SIZES = [64, 80, 96, 112, 128, 256]
24
BLOCK_SIZES = [16, 32]
25
USE_ALIBI = [False, True]
26
SEEDS = [0]
27
+DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
28
29
30
def ref_masked_attention(
@@ -87,7 +88,7 @@ def ref_single_query_cached_kv_attention(
88
alibi_bias = None
89
if alibi_slopes is not None:
90
# Create the ALiBi bias used in the paged attention kernel.
90
- position_ids = torch.arange(context_len, device="cuda").int()
91
+ position_ids = torch.arange(context_len, device=query.device).int()
92
alibi_bias = (position_ids - context_len + 1).float()
93
alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
94
1, 1, -1)
@@ -105,6 +106,7 @@ def ref_single_query_cached_kv_attention(
106
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
107
@pytest.mark.parametrize("dtype", DTYPES)
108
@pytest.mark.parametrize("seed", SEEDS)
109
+@pytest.mark.parametrize("device", DEVICES)
110
def test_paged_attention(
111
kv_cache_factory,
112
version: str,
@@ -115,18 +117,19 @@ def test_paged_attention(
117
block_size: int,
118
dtype: torch.dtype,
119
seed: int,
120
+ device: int,
121
) -> None:
122
random.seed(seed)
123
torch.random.manual_seed(seed)
124
torch.cuda.manual_seed(seed)
122
-
125
+ gpu_id = f"cuda:{device}"
126
scale = float(1.0 / (head_size**0.5))
127
num_query_heads, num_kv_heads = num_heads
128
query = torch.empty(num_seqs,
129
num_query_heads,
130
head_size,
131
dtype=dtype,
129
- device="cuda")
132
+ device=gpu_id)
133
query.uniform_(-scale, scale)
134
135
assert num_query_heads % num_kv_heads == 0
@@ -135,12 +138,12 @@ def test_paged_attention(
138
if use_alibi:
139
alibi_slopes = torch.randn(num_query_heads,
140
dtype=torch.float,
138
- device="cuda")
141
+ device=gpu_id)
142
143
context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
144
context_lens[-1] = MAX_SEQ_LEN
145
max_context_len = max(context_lens)
143
- context_lens = torch.tensor(context_lens, dtype=torch.int, device="cuda")
146
+ context_lens = torch.tensor(context_lens, dtype=torch.int, device=gpu_id)
147
148
# Create the block tables.
149
max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
@@ -151,12 +154,12 @@ def test_paged_attention(
154
for _ in range(max_num_blocks_per_seq)
155
]
156
block_tables.append(block_table)
154
- block_tables = torch.tensor(block_tables, dtype=torch.int, device="cuda")
157
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device=gpu_id)
158
159
# Create the KV caches.
160
key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
161
num_kv_heads, head_size, dtype,
159
- seed)
162
+ seed, gpu_id)
163
key_cache, value_cache = key_caches[0], value_caches[0]
164
165
# Call the paged attention kernel.
@@ -249,7 +252,7 @@ def ref_multi_query_kv_attention(
252
attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
253
diagonal=1)
254
attn_mask = attn_mask * torch.finfo(dtype).min
252
- attn_mask = attn_mask.to(dtype=dtype, device="cuda")
255
+ attn_mask = attn_mask.to(dtype=dtype, device=query.device)
256
257
ref_output = ref_masked_attention(
258
query[start_idx:end_idx],
@@ -269,6 +272,7 @@ def ref_multi_query_kv_attention(
272
@pytest.mark.parametrize("head_size", HEAD_SIZES)
273
@pytest.mark.parametrize("dtype", DTYPES)
274
@pytest.mark.parametrize("seed", SEEDS)
275
+@pytest.mark.parametrize("device", DEVICES)
276
@torch.inference_mode()
277
def test_multi_query_kv_attention(
278
num_seqs: int,
@@ -276,11 +280,12 @@ def test_multi_query_kv_attention(
280
head_size: int,
281
dtype: torch.dtype,
282
seed: int,
283
+ device: int,
284
) -> None:
285
random.seed(seed)
286
torch.random.manual_seed(seed)
287
torch.cuda.manual_seed(seed)
283
-
288
+ gpu_id = f"cuda:{device}"
289
# MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
290
# As the xformers library is already tested with its own tests, we can use
291
# a smaller MAX_SEQ_LEN here.
@@ -294,7 +299,7 @@ def test_multi_query_kv_attention(
299
num_query_heads + 2 * num_kv_heads,
300
head_size,
301
dtype=dtype,
297
- device="cuda")
302
+ device=gpu_id)
303
qkv.uniform_(-scale, scale)
304
query, key, value = qkv.split(
305
[num_query_heads, num_kv_heads, num_kv_heads], dim=1)
tests/kernels/test_cache.py
+11
-6
@@ -14,6 +14,7 @@ BLOCK_SIZES = [8, 16, 32]
14
NUM_BLOCKS = [1024, 36000] # Arbitrary values for testing
15
NUM_MAPPINGS = [256] # Arbitrary values for testing
16
SEEDS = [0]
17
+DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
18
19
20
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@@ -24,6 +25,7 @@ SEEDS = [0]
25
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
26
@pytest.mark.parametrize("dtype", DTYPES)
27
@pytest.mark.parametrize("seed", SEEDS)
28
+@pytest.mark.parametrize("device", DEVICES)
29
@torch.inference_mode()
30
def test_copy_blocks(
31
kv_cache_factory,
@@ -35,11 +37,12 @@ def test_copy_blocks(
37
num_blocks: int,
38
dtype: torch.dtype,
39
seed: int,
40
+ device: int,
41
) -> None:
42
random.seed(seed)
43
torch.random.manual_seed(seed)
44
torch.cuda.manual_seed(seed)
42
-
45
+ gpu_id = f"cuda:{device}"
46
# Generate random block mappings where each source block is mapped to two
47
# destination blocks.
48
assert 2 * num_mappings <= num_blocks
@@ -56,7 +59,7 @@ def test_copy_blocks(
59
# Create the KV caches.
60
key_caches, value_caches = kv_cache_factory(num_blocks, block_size,
61
num_layers, num_heads,
59
- head_size, dtype, seed)
62
+ head_size, dtype, seed, gpu_id)
63
64
# Clone the KV caches.
65
cloned_key_caches = [key_cache.clone() for key_cache in key_caches]
@@ -88,6 +91,7 @@ def test_copy_blocks(
91
@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
92
@pytest.mark.parametrize("dtype", DTYPES)
93
@pytest.mark.parametrize("seed", SEEDS)
94
+@pytest.mark.parametrize("device", DEVICES)
95
@torch.inference_mode()
96
def test_reshape_and_cache(
97
kv_cache_factory,
@@ -98,28 +102,29 @@ def test_reshape_and_cache(
102
num_blocks: int,
103
dtype: torch.dtype,
104
seed: int,
105
+ device: int,
106
) -> None:
107
random.seed(seed)
108
torch.random.manual_seed(seed)
109
torch.cuda.manual_seed(seed)
105
-
110
+ gpu_id = f"cuda:{device}"
111
# Create a random slot mapping.
112
num_slots = block_size * num_blocks
113
slot_mapping = random.sample(range(num_slots), num_tokens)
109
- slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device="cuda")
114
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device=gpu_id)
115
116
qkv = torch.randn(num_tokens,
117
3,
118
num_heads,
119
head_size,
120
dtype=dtype,
116
- device="cuda")
121
+ device=gpu_id)
122
_, key, value = qkv.unbind(dim=1)
123
124
# Create the KV caches.
125
key_caches, value_caches = kv_cache_factory(num_blocks, block_size, 1,
126
num_heads, head_size, dtype,
122
- seed)
127
+ seed, gpu_id)
128
key_cache, value_cache = key_caches[0], value_caches[0]
129
130
# Clone the KV caches.
tests/kernels/test_layernorm.py
+6
-3
@@ -8,6 +8,7 @@ NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8
HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
9
ADD_RESIDUAL = [False, True]
10
SEEDS = [0]
11
+DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
12
13
14
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@@ -15,6 +16,7 @@ SEEDS = [0]
16
@pytest.mark.parametrize("add_residual", ADD_RESIDUAL)
17
@pytest.mark.parametrize("dtype", DTYPES)
18
@pytest.mark.parametrize("seed", SEEDS)
19
+@pytest.mark.parametrize("device", DEVICES)
20
@torch.inference_mode()
21
def test_rms_norm(
22
num_tokens: int,
@@ -22,14 +24,15 @@ def test_rms_norm(
24
add_residual: bool,
25
dtype: torch.dtype,
26
seed: int,
27
+ device: int,
28
) -> None:
29
torch.random.manual_seed(seed)
30
torch.cuda.manual_seed(seed)
28
-
29
- layer = RMSNorm(hidden_size).to(dtype).cuda()
31
+ gpu_id = f"cuda:{device}"
32
+ layer = RMSNorm(hidden_size).to(dtype=dtype, device=gpu_id)
33
layer.weight.data.normal_(mean=1.0, std=0.1)
34
scale = 1 / (2 * hidden_size)
32
- x = torch.randn(num_tokens, hidden_size, dtype=dtype, device="cuda")
35
+ x = torch.randn(num_tokens, hidden_size, dtype=dtype, device=gpu_id)
36
x *= scale
37
residual = torch.randn_like(x) * scale if add_residual else None
38
tests/kernels/test_pos_encoding.py
+7
-4
@@ -13,6 +13,7 @@ NUM_HEADS = [7, 17] # Arbitrary values for testing
13
BATCH_SIZES = [1, 5] # Arbitrary values for testing
14
SEQ_LENS = [11, 8192] # Arbitrary values for testing
15
SEEDS = [0]
16
+DEVICES = [i for i in range(1 if torch.cuda.device_count() == 1 else 2)]
17
18
19
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
@@ -23,6 +24,7 @@ SEEDS = [0]
24
@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
25
@pytest.mark.parametrize("dtype", DTYPES)
26
@pytest.mark.parametrize("seed", SEEDS)
27
+@pytest.mark.parametrize("device", DEVICES)
28
@torch.inference_mode()
29
def test_rotary_embedding(
30
is_neox_style: bool,
@@ -33,6 +35,7 @@ def test_rotary_embedding(
35
rotary_dim: Optional[int],
36
dtype: torch.dtype,
37
seed: int,
38
+ device: int,
39
max_position: int = 8192,
40
base: int = 10000,
41
) -> None:
@@ -40,20 +43,20 @@ def test_rotary_embedding(
43
rotary_dim = head_size
44
torch.random.manual_seed(seed)
45
torch.cuda.manual_seed(seed)
43
-
46
+ gpu_id = f"cuda:{device}"
47
if rotary_dim is None:
48
rotary_dim = head_size
49
rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style)
47
- rope = rope.to(dtype).cuda()
50
+ rope = rope.to(dtype=dtype, device=gpu_id)
51
52
positions = torch.randint(0,
53
max_position, (batch_size, seq_len),
51
- device="cuda")
54
+ device=gpu_id)
55
query = torch.randn(batch_size,
56
seq_len,
57
num_heads * head_size,
58
dtype=dtype,
56
- device="cuda")
59
+ device=gpu_id)
60
key = torch.randn_like(query)
61
62
# NOTE(woosuk): The reference implementation should be executed first