@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ce8647dc5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix integer overflows in attention & cache ops (#1514)

Woosuk Kwon committed Oct 31, 2023 at 15:19 UTC 0ce8647dc5bc89074541447d8d410aae0d912173
5 files changed +53 -47
csrc/attention/attention_kernels.cu
+8 -2
@@ -175,7 +175,10 @@ __device__ void paged_attention_kernel(
175 // dot product with the query.
176 const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
177 for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
178 - const int physical_block_number = block_table[block_idx];
178 + // NOTE(woosuk): The block number is stored in int32. However, we cast it to int64
179 + // because int32 can lead to overflow when this variable is multiplied by large numbers
180 + // (e.g., kv_block_stride).
181 + const int64_t physical_block_number = static_cast<int64_t>(block_table[block_idx]);
182
183 // Load a key to registers.
184 // Each thread in a thread group has a different part of the key.
@@ -285,7 +288,10 @@ __device__ void paged_attention_kernel(
288 scalar_t zero_value;
289 zero(zero_value);
290 for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx; block_idx += NUM_WARPS) {
288 - const int physical_block_number = block_table[block_idx];
291 + // NOTE(woosuk): The block number is stored in int32. However, we cast it to int64
292 + // because int32 can lead to overflow when this variable is multiplied by large numbers
293 + // (e.g., kv_block_stride).
294 + const int64_t physical_block_number = static_cast<int64_t>(block_table[block_idx]);
295 const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
296 const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
297 L_vec logits_vec;
csrc/cache_kernels.cu
+36 -36
@@ -55,26 +55,26 @@ template<typename scalar_t>
55 __global__ void copy_blocks_kernel(
56 int64_t* key_cache_ptrs,
57 int64_t* value_cache_ptrs,
58 - const int* __restrict__ block_mapping,
58 + const int64_t* __restrict__ block_mapping,
59 const int numel_per_block) {
60 const int layer_idx = blockIdx.x;
61 const int pair_idx = blockIdx.y;
62
63 scalar_t* key_cache = reinterpret_cast<scalar_t*>(key_cache_ptrs[layer_idx]);
64 scalar_t* value_cache = reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
65 - int src_block_number = block_mapping[2 * pair_idx];
66 - int dst_block_number = block_mapping[2 * pair_idx + 1];
65 + int64_t src_block_number = block_mapping[2 * pair_idx];
66 + int64_t dst_block_number = block_mapping[2 * pair_idx + 1];
67
68 - const int src_block_offset = src_block_number * numel_per_block;
69 - const int dst_block_offset = dst_block_number * numel_per_block;
68 + const int64_t src_block_offset = src_block_number * numel_per_block;
69 + const int64_t dst_block_offset = dst_block_number * numel_per_block;
70 for (int i = threadIdx.x; i < numel_per_block; i += blockDim.x) {
71 - int src_offset = src_block_offset + i;
72 - int dst_offset = dst_block_offset + i;
71 + int64_t src_offset = src_block_offset + i;
72 + int64_t dst_offset = dst_block_offset + i;
73 key_cache[dst_offset] = key_cache[src_offset];
74 }
75 for (int i = threadIdx.x; i < numel_per_block; i += blockDim.x) {
76 - int src_offset = src_block_offset + i;
77 - int dst_offset = dst_block_offset + i;
76 + int64_t src_offset = src_block_offset + i;
77 + int64_t dst_offset = dst_block_offset + i;
78 value_cache[dst_offset] = value_cache[src_offset];
79 }
80 }
@@ -102,15 +102,15 @@ void copy_blocks(
102 value_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
103 }
104 // Create block mapping array.
105 - std::vector<int> block_mapping_vec;
105 + std::vector<int64_t> block_mapping_vec;
106 for (const auto& pair : block_mapping) {
107 - int src_block_number = pair.first;
108 - for (int dst_block_number : pair.second) {
107 + int64_t src_block_number = pair.first;
108 + for (int64_t dst_block_number : pair.second) {
109 block_mapping_vec.push_back(src_block_number);
110 block_mapping_vec.push_back(dst_block_number);
111 }
112 }
113 - int* block_mapping_array = block_mapping_vec.data();
113 + int64_t* block_mapping_array = block_mapping_vec.data();
114 int num_pairs = block_mapping_vec.size() / 2;
115
116 // Move the data structures to the GPU.
@@ -120,7 +120,7 @@ void copy_blocks(
120 torch::Tensor value_cache_ptrs_tensor = torch::from_blob(
121 value_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
122 torch::Tensor block_mapping_tensor = torch::from_blob(
123 - block_mapping_array, {2 * num_pairs}, torch::kInt).to(cache_device);
123 + block_mapping_array, {2 * num_pairs}, torch::kInt64).to(cache_device);
124
125 // Launch the kernel.
126 const int numel_per_block = key_caches[0][0].numel();
@@ -132,7 +132,7 @@ void copy_blocks(
132 vllm::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
133 key_cache_ptrs_tensor.data_ptr<int64_t>(),
134 value_cache_ptrs_tensor.data_ptr<int64_t>(),
135 - block_mapping_tensor.data_ptr<int>(),
135 + block_mapping_tensor.data_ptr<int64_t>(),
136 numel_per_block);
137 }));
138 }
@@ -141,46 +141,46 @@ namespace vllm {
141
142 template<typename scalar_t>
143 __global__ void reshape_and_cache_kernel(
144 - const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
145 - const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
146 - scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
147 - scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
148 - const int* __restrict__ slot_mapping, // [num_tokens]
144 + const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
145 + const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
146 + scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
147 + scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
148 + const int64_t* __restrict__ slot_mapping, // [num_tokens]
149 const int key_stride,
150 const int value_stride,
151 const int num_heads,
152 const int head_size,
153 const int block_size,
154 const int x) {
155 - const int token_idx = blockIdx.x;
156 - const int slot_idx = slot_mapping[token_idx];
155 + const int64_t token_idx = blockIdx.x;
156 + const int64_t slot_idx = slot_mapping[token_idx];
157 if (slot_idx < 0) {
158 // Padding token that should be ignored.
159 return;
160 }
161
162 - const int block_idx = slot_idx / block_size;
163 - const int block_offset = slot_idx % block_size;
162 + const int64_t block_idx = slot_idx / block_size;
163 + const int64_t block_offset = slot_idx % block_size;
164
165 const int n = num_heads * head_size;
166 for (int i = threadIdx.x; i < n; i += blockDim.x) {
167 - const int src_key_idx = token_idx * key_stride + i;
168 - const int src_value_idx = token_idx * value_stride + i;
167 + const int64_t src_key_idx = token_idx * key_stride + i;
168 + const int64_t src_value_idx = token_idx * value_stride + i;
169
170 const int head_idx = i / head_size;
171 const int head_offset = i % head_size;
172 const int x_idx = head_offset / x;
173 const int x_offset = head_offset % x;
174
175 - const int tgt_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
176 - + head_idx * (head_size / x) * block_size * x
177 - + x_idx * block_size * x
178 - + block_offset * x
179 - + x_offset;
180 - const int tgt_value_idx = block_idx * num_heads * head_size * block_size
181 - + head_idx * head_size * block_size
182 - + head_offset * block_size
183 - + block_offset;
175 + const int64_t tgt_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
176 + + head_idx * (head_size / x) * block_size * x
177 + + x_idx * block_size * x
178 + + block_offset * x
179 + + x_offset;
180 + const int64_t tgt_value_idx = block_idx * num_heads * head_size * block_size
181 + + head_idx * head_size * block_size
182 + + head_offset * block_size
183 + + block_offset;
184 key_cache[tgt_key_idx] = key[src_key_idx];
185 value_cache[tgt_value_idx] = value[src_value_idx];
186 }
@@ -216,7 +216,7 @@ void reshape_and_cache(
216 value.data_ptr<scalar_t>(),
217 key_cache.data_ptr<scalar_t>(),
218 value_cache.data_ptr<scalar_t>(),
219 - slot_mapping.data_ptr<int>(),
219 + slot_mapping.data_ptr<int64_t>(),
220 key_stride,
221 value_stride,
222 num_heads,
tests/kernels/test_attention.py
+1 -1
@@ -13,7 +13,7 @@ FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
13 # This will change depending on the compute capability.
14 # - 512 as a buffer
15 MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
16 -NUM_BLOCKS = 128 # Arbitrary values for testing
16 +NUM_BLOCKS = 40000 # Arbitrary values for testing
17 PARTITION_SIZE = 512
18
19 DTYPES = [torch.half, torch.bfloat16, torch.float]
tests/kernels/test_cache.py
+7 -7
@@ -6,13 +6,13 @@ import torch
6 from vllm import cache_ops
7
8 DTYPES = [torch.half, torch.bfloat16, torch.float]
9 -NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
10 -NUM_LAYERS = [5] # Arbitrary values for testing
9 +NUM_TOKENS = [83] # Arbitrary values for testing
10 +NUM_LAYERS = [1] # Arbitrary values for testing
11 NUM_HEADS = [8] # Arbitrary values for testing
12 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13 BLOCK_SIZES = [8, 16, 32]
14 -NUM_BLOCKS = [1024] # Arbitrary values for testing
15 -NUM_MAPPINGS = [32, 256] # Arbitrary values for testing
14 +NUM_BLOCKS = [1024, 36000] # Arbitrary values for testing
15 +NUM_MAPPINGS = [256] # Arbitrary values for testing
16 SEEDS = [0]
17
18
@@ -69,9 +69,9 @@ def test_copy_blocks(
69 for src, dsts in block_mapping.items():
70 for dst in dsts:
71 for cloned_key_cache in cloned_key_caches:
72 - cloned_key_cache[dst] = cloned_key_cache[src]
72 + cloned_key_cache[dst].copy_(cloned_key_cache[src])
73 for cloned_value_cache in cloned_value_caches:
74 - cloned_value_cache[dst] = cloned_value_cache[src]
74 + cloned_value_cache[dst].copy_(cloned_value_cache[src])
75
76 # Compare the results.
77 for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
@@ -106,7 +106,7 @@ def test_reshape_and_cache(
106 # Create a random slot mapping.
107 num_slots = block_size * num_blocks
108 slot_mapping = random.sample(range(num_slots), num_tokens)
109 - slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device="cuda")
109 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device="cuda")
110
111 qkv = torch.randn(num_tokens,
112 3,
vllm/worker/worker.py
+1 -1
@@ -301,7 +301,7 @@ class Worker:
301 dtype=torch.long,
302 device="cuda")
303 slot_mapping_tensor = torch.tensor(padded_slot_mapping,
304 - dtype=torch.int,
304 + dtype=torch.long,
305 device="cuda")
306 context_lens_tensor = torch.tensor(context_lens,
307 dtype=torch.int,