A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Kernel] Make rotary_embedding ops more flexible with input shape (#12777)
Isotr0py committed
Feb 7, 2025 at 00:46 UTC
85ac82d228ef6af4e8fc6332d918133e783a0fdb
4 files changed
+115
-57
csrc/pos_encoding_kernels.cu
+89
-14
@@ -124,18 +124,54 @@ __global__ void batched_rotary_embedding_kernel(
124
void rotary_embedding(
125
torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
126
torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or
127
- // [num_tokens, num_heads * head_size]
127
+ // [num_tokens, num_heads * head_size] or
128
+ // [batch_size, seq_len, num_heads, head_size] or
129
+ // [num_tokens, num_heads, head_size]
130
torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
129
- // [num_tokens, num_kv_heads * head_size]
131
+ // [num_tokens, num_kv_heads * head_size] or
132
+ // [batch_size, seq_len, num_heads, head_size] or
133
+ // [num_tokens, num_heads, head_size]
134
int64_t head_size,
135
torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
136
bool is_neox) {
133
- int64_t num_tokens = query.numel() / query.size(-1);
137
+ // num_tokens = batch_size * seq_len
138
+ int64_t num_tokens = positions.numel();
139
+ int positions_ndim = positions.dim();
140
+
141
+ // Make sure num_tokens dim is consistent across positions, query, and key.
142
+ TORCH_CHECK(
143
+ positions_ndim == 1 || positions_ndim == 2,
144
+ "positions must have shape [num_tokens] or [batch_size, seq_len]");
145
+ if (positions_ndim == 1) {
146
+ TORCH_CHECK(
147
+ query.size(0) == positions.size(0) && key.size(0) == positions.size(0),
148
+ "query, key and positions must have the same number of tokens");
149
+ }
150
+ if (positions_ndim == 2) {
151
+ TORCH_CHECK(
152
+ query.size(0) == positions.size(0) &&
153
+ key.size(0) == positions.size(0) &&
154
+ query.size(1) == positions.size(1) &&
155
+ key.size(1) == positions.size(1),
156
+ "query, key and positions must have the same batch_size and seq_len");
157
+ }
158
+
159
+ // Make sure head_size is valid for query and key
160
+ // hidden_size = num_heads * head_size
161
+ int query_hidden_size = query.numel() / num_tokens;
162
+ int key_hidden_size = key.numel() / num_tokens;
163
+ TORCH_CHECK(query_hidden_size % head_size == 0);
164
+ TORCH_CHECK(key_hidden_size % head_size == 0);
165
+
166
+ // Make sure query and key have consistent number of heads
167
+ int num_heads = query_hidden_size / head_size;
168
+ int num_kv_heads = key_hidden_size / head_size;
169
+ TORCH_CHECK(num_heads % num_kv_heads == 0);
170
+
171
int rot_dim = cos_sin_cache.size(1);
135
- int num_heads = query.size(-1) / head_size;
136
- int num_kv_heads = key.size(-1) / head_size;
137
- int64_t query_stride = query.stride(-2);
138
- int64_t key_stride = key.stride(-2);
172
+ int seq_dim_idx = positions_ndim - 1;
173
+ int64_t query_stride = query.stride(seq_dim_idx);
174
+ int64_t key_stride = key.stride(seq_dim_idx);
175
176
dim3 grid(num_tokens);
177
dim3 block(std::min<int64_t>(num_heads * rot_dim / 2, 512));
@@ -165,19 +201,58 @@ and process in batched manner.
201
void batched_rotary_embedding(
202
torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
203
torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or
168
- // [num_tokens, num_heads * head_size]
204
+ // [num_tokens, num_heads * head_size] or
205
+ // [batch_size, seq_len, num_heads, head_size] or
206
+ // [num_tokens, num_heads, head_size]
207
torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or
170
- // [num_tokens, num_kv_heads * head_size]
208
+ // [num_tokens, num_kv_heads * head_size] or
209
+ // [batch_size, seq_len, num_heads, head_size] or
210
+ // [num_tokens, num_heads, head_size]
211
int64_t head_size,
212
torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
213
bool is_neox, int64_t rot_dim,
174
- torch::Tensor& cos_sin_cache_offsets // [num_tokens]
214
+ torch::Tensor& cos_sin_cache_offsets // [num_tokens] or [batch_size]
215
) {
216
+ // num_tokens = batch_size * seq_len
217
int64_t num_tokens = cos_sin_cache_offsets.size(0);
177
- int num_heads = query.size(-1) / head_size;
178
- int num_kv_heads = key.size(-1) / head_size;
179
- int64_t query_stride = query.stride(-2);
180
- int64_t key_stride = key.stride(-2);
218
+ TORCH_CHECK(
219
+ positions.size(0) == num_tokens || positions.numel() == num_tokens,
220
+ "positions must have the same num_tokens or batch_size as "
221
+ "cos_sin_cache_offsets");
222
+
223
+ int positions_ndim = positions.dim();
224
+ // Make sure num_tokens dim is consistent across positions, query, and key.
225
+ TORCH_CHECK(
226
+ positions_ndim == 1 || positions_ndim == 2,
227
+ "positions must have shape [num_tokens] or [batch_size, seq_len]");
228
+ if (positions_ndim == 1) {
229
+ TORCH_CHECK(
230
+ query.size(0) == positions.size(0) && key.size(0) == positions.size(0),
231
+ "query, key and positions must have the same number of tokens");
232
+ }
233
+ if (positions_ndim == 2) {
234
+ TORCH_CHECK(
235
+ query.size(0) == positions.size(0) &&
236
+ key.size(0) == positions.size(0) &&
237
+ query.size(1) == positions.size(1) &&
238
+ key.size(1) == positions.size(1),
239
+ "query, key and positions must have the same batch_size and seq_len");
240
+ }
241
+
242
+ // Make sure head_size is valid for query and key
243
+ int query_hidden_size = query.numel() / num_tokens;
244
+ int key_hidden_size = key.numel() / num_tokens;
245
+ TORCH_CHECK(query_hidden_size % head_size == 0);
246
+ TORCH_CHECK(key_hidden_size % head_size == 0);
247
+
248
+ // Make sure query and key have concistent number of heads
249
+ int num_heads = query_hidden_size / head_size;
250
+ int num_kv_heads = key_hidden_size / head_size;
251
+ TORCH_CHECK(num_heads % num_kv_heads == 0);
252
+
253
+ int seq_dim_idx = positions_ndim - 1;
254
+ int64_t query_stride = query.stride(seq_dim_idx);
255
+ int64_t key_stride = key.stride(seq_dim_idx);
256
257
dim3 grid(num_tokens);
258
dim3 block(std::min<int64_t>(num_heads * rot_dim / 2, 512));
tests/kernels/test_pos_encoding.py
+22
-9
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
from itertools import accumulate, product
4
-from typing import Dict, List, Optional
4
+from typing import Callable, Dict, List, Optional
5
6
import pytest
7
import torch
@@ -24,7 +24,21 @@ CUDA_DEVICES = [
24
]
25
26
27
+def _get_flat_tensor_shape(batch_size: int, seq_len: int, num_heads: int,
28
+ head_size: int) -> tuple[int, ...]:
29
+ return (batch_size, seq_len, num_heads * head_size)
30
+
31
+
32
+def _get_batch_tensor_shape(batch_size: int, seq_len: int, num_heads: int,
33
+ head_size: int) -> tuple[int, ...]:
34
+ return (batch_size, seq_len, num_heads, head_size)
35
+
36
+
37
+TENSORS_SHAPES_FN = [_get_batch_tensor_shape, _get_flat_tensor_shape]
38
+
39
+
40
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
41
+@pytest.mark.parametrize("tensor_shape_fn", TENSORS_SHAPES_FN)
42
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
43
@pytest.mark.parametrize("seq_len", SEQ_LENS)
44
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -36,6 +50,7 @@ CUDA_DEVICES = [
50
@torch.inference_mode()
51
def test_rotary_embedding(
52
is_neox_style: bool,
53
+ tensor_shape_fn: Callable[[int, int, int, int], tuple[int]],
54
batch_size: int,
55
seq_len: int,
56
num_heads: int,
@@ -58,10 +73,8 @@ def test_rotary_embedding(
73
rope = rope.to(dtype=dtype)
74
75
positions = torch.randint(0, max_position, (batch_size, seq_len))
61
- query = torch.randn(batch_size,
62
- seq_len,
63
- num_heads * head_size,
64
- dtype=dtype)
76
+ query_shape = tensor_shape_fn(batch_size, seq_len, num_heads, head_size)
77
+ query = torch.randn(query_shape, dtype=dtype)
78
key = torch.randn_like(query)
79
80
# NOTE(woosuk): The reference implementation should be executed first
@@ -80,6 +93,7 @@ def test_rotary_embedding(
93
94
95
@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
96
+@pytest.mark.parametrize("tensor_shape_fn", TENSORS_SHAPES_FN)
97
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
98
@pytest.mark.parametrize("seq_len", SEQ_LENS)
99
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -91,6 +105,7 @@ def test_rotary_embedding(
105
@torch.inference_mode()
106
def test_batched_rotary_embedding(
107
is_neox_style: bool,
108
+ tensor_shape_fn: Callable[[int, int, int, int], tuple[int]],
109
batch_size: int,
110
seq_len: int,
111
num_heads: int,
@@ -113,10 +128,8 @@ def test_batched_rotary_embedding(
128
rope = rope.to(dtype=dtype)
129
130
positions = torch.randint(0, max_position, (batch_size, seq_len))
116
- query = torch.randn(batch_size,
117
- seq_len,
118
- num_heads * head_size,
119
- dtype=dtype)
131
+ query_shape = tensor_shape_fn(batch_size, seq_len, num_heads, head_size)
132
+ query = torch.randn(query_shape, dtype=dtype)
133
key = torch.randn_like(query)
134
135
# NOTE(woosuk): The reference implementation should be executed first
vllm/attention/backends/mla/utils.py
+3
-22
@@ -424,24 +424,6 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
424
) -> torch.Tensor:
425
raise NotImplementedError
426
427
- def apply_pure_rope(
428
- self,
429
- input_positions: torch.Tensor,
430
- q_pe: torch.Tensor,
431
- k_pe: torch.Tensor,
432
- ) -> tuple[torch.Tensor, torch.Tensor]:
433
- seq_len = input_positions.size(0)
434
- ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
435
-
436
- q_pe, k_pe = self.rotary_emb(
437
- input_positions,
438
- q_pe.reshape(seq_len, -1),
439
- k_pe.reshape(seq_len, -1),
440
- )
441
- q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
442
-
443
- return q_pe, k_pe
444
-
427
def forward(
428
self,
429
layer: AttentionLayer,
@@ -466,14 +448,13 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
448
# Restore head dim (for rotary embedding)
449
k_pe = k_pe.unsqueeze(1)
450
assert hasattr(attn_metadata, "input_positions")
469
- rope_fn = (self.rotary_emb
470
- if self.use_yarn_rope else self.apply_pure_rope)
451
452
if is_decode:
453
q_nope = self._q_proj_and_k_up_proj(hidden_states_or_q_c)
454
q_pe = torch.matmul(hidden_states_or_q_c, self.W_QR)\
455
.view(-1, self.num_heads, self.qk_rope_head_dim)
476
- q_pe, k_pe = rope_fn(attn_metadata.input_positions, q_pe, k_pe)
456
+ q_pe, k_pe = self.rotary_emb(attn_metadata.input_positions, q_pe,
457
+ k_pe)
458
else:
459
assert is_prefill
460
q = self.q_proj(hidden_states_or_q_c)[0]\
@@ -481,7 +462,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
462
463
# TODO(lucas): there must be a nicer way to write this line
464
q[..., self.qk_nope_head_dim:], k_pe = \
484
- rope_fn(
465
+ self.rotary_emb(
466
attn_metadata.input_positions,
467
q[..., self.qk_nope_head_dim:], k_pe)
468
vllm/model_executor/models/deepseek_v2.py
+1
-12
@@ -257,9 +257,7 @@ class DeepseekV2Attention(nn.Module):
257
prefix=f"{prefix}.o_proj")
258
if rope_scaling:
259
rope_scaling["rope_type"] = 'deepseek_yarn'
260
- self.use_normal_rope = False
261
- else:
262
- self.use_normal_rope = True
260
+
261
self.rotary_emb = get_rope(qk_rope_head_dim,
262
rotary_dim=qk_rope_head_dim,
263
max_position=max_position_embeddings,
@@ -309,17 +307,8 @@ class DeepseekV2Attention(nn.Module):
307
k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
308
k_pe = latent_cache[:, :, self.kv_lora_rank:]
309
312
- if self.use_normal_rope:
313
- seq_len = positions.size(0)
314
- ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
315
- q_pe = q_pe.reshape(seq_len, -1)
316
- k_pe = k_pe.reshape(seq_len, -1)
317
-
310
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
311
320
- if self.use_normal_rope:
321
- q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
322
-
312
q[..., self.qk_nope_head_dim:] = q_pe
313
k = torch.empty_like(q)
314
k[..., :self.qk_nope_head_dim] = k_nope