@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 76a7983b23
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BugFix] Fix RoPE kernel on long sequences(#2164)

Woosuk Kwon committed Dec 17, 2023 at 17:09 UTC 76a7983b23f55785005d361e20784f7d3d6e11ce
1 file changed +6 -6
csrc/pos_encoding_kernels.cu
+6 -6
@@ -43,8 +43,8 @@ __global__ void rotary_embedding_kernel(
43 scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
44 const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
45 const int rot_dim,
46 - const int query_stride,
47 - const int key_stride,
46 + const int64_t query_stride,
47 + const int64_t key_stride,
48 const int num_heads,
49 const int num_kv_heads,
50 const int head_size) {
@@ -60,7 +60,7 @@ __global__ void rotary_embedding_kernel(
60 const int nq = num_heads * embed_dim;
61 for (int i = threadIdx.x; i < nq; i += blockDim.x) {
62 const int head_idx = i / embed_dim;
63 - const int token_head = token_idx * query_stride + head_idx * head_size;
63 + const int64_t token_head = token_idx * query_stride + head_idx * head_size;
64 const int rot_offset = i % embed_dim;
65 apply_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
66 sin_ptr, rot_offset, embed_dim);
@@ -69,7 +69,7 @@ __global__ void rotary_embedding_kernel(
69 const int nk = num_kv_heads * embed_dim;
70 for (int i = threadIdx.x; i < nk; i += blockDim.x) {
71 const int head_idx = i / embed_dim;
72 - const int token_head = token_idx * key_stride + head_idx * head_size;
72 + const int64_t token_head = token_idx * key_stride + head_idx * head_size;
73 const int rot_offset = i % embed_dim;
74 apply_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
75 sin_ptr, rot_offset, embed_dim);
@@ -89,8 +89,8 @@ void rotary_embedding(
89 int rot_dim = cos_sin_cache.size(1);
90 int num_heads = query.size(-1) / head_size;
91 int num_kv_heads = key.size(-1) / head_size;
92 - int query_stride = query.stride(-2);
93 - int key_stride = key.stride(-2);
92 + int64_t query_stride = query.stride(-2);
93 + int64_t key_stride = key.stride(-2);
94
95 dim3 grid(num_tokens);
96 dim3 block(std::min(num_heads * rot_dim / 2, 512));