A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Core] Flashinfer - Remove advance step size restriction (#10282)
Pavani Majety committed
Nov 13, 2024 at 00:29 UTC
b6dde330198848a4a9903c1f0f97c3235fba0ba0
1 file changed
+38
-28
csrc/prepare_inputs/advance_step.cu
+38
-28
@@ -88,6 +88,7 @@ inline void verify_tensor(std::string const& name, torch::Tensor const& t,
88
}
89
}
90
91
+/// each thread processes a block per query
92
__global__ void advance_step_flashinfer_kernel(
93
int num_threads, int num_seqs, int num_queries, int block_size,
94
long* input_tokens_ptr, long const* sampled_token_ids_ptr,
@@ -134,8 +135,10 @@ __global__ void advance_step_flashinfer_indptr_kernel(
135
int num_threads, int num_seqs, int num_queries, int* paged_kv_indptr_ptr,
136
int* block_table_bound_ptr) {
137
int idx = blockIdx.x * num_threads + threadIdx.x;
137
-
138
// Update paged_kv_indptr
139
+ if (idx == 0) {
140
+ paged_kv_indptr_ptr[idx] = 0;
141
+ }
142
if (idx < num_queries) {
143
int sum = 0;
144
for (int i = 0; i <= idx; ++i) {
@@ -146,20 +149,33 @@ __global__ void advance_step_flashinfer_indptr_kernel(
149
}
150
151
__global__ void advance_step_flashinfer_indices_kernel(
149
- int num_threads, int num_seqs, int num_queries, int const* block_tables_ptr,
150
- int64_t const block_tables_stride, int* paged_kv_indices_ptr,
152
+ int num_seqs, int num_queries, int const* block_tables_ptr,
153
+ int64_t const max_num_blocks_per_seq, int* paged_kv_indices_ptr,
154
int* paged_kv_indptr_ptr, int* block_table_bound_ptr) {
152
- int idx = blockIdx.x * num_threads + threadIdx.x;
153
- int row = idx / block_tables_stride;
154
- int col = idx % block_tables_stride;
155
-
156
- if (row < num_queries && col < block_table_bound_ptr[row]) {
157
- paged_kv_indices_ptr[paged_kv_indptr_ptr[row] + col] =
158
- block_tables_ptr[row * block_tables_stride + col];
155
+ // note: max_num_blocks_per_seq = block_tables.stride(0)
156
+ int tid = blockIdx.x * blockDim.x + threadIdx.x;
157
+
158
+ // when cuda graphs are enabled, paged_kv_indptr tensor
159
+ // has to be updated for the padded queries
160
+ // tid represents a query# for paged_kv_indptr tensor
161
+ if (num_queries < tid && tid <= num_seqs) {
162
+ paged_kv_indptr_ptr[tid] = paged_kv_indptr_ptr[num_queries];
163
}
160
- // if cudagraph, fill padded seqs with the last valid seq's indptr
161
- if (num_queries < row && row <= num_seqs) {
162
- paged_kv_indptr_ptr[row] = paged_kv_indptr_ptr[num_queries];
164
+
165
+ // each thread processes a block_ptr in block_tables
166
+ // block_tables shape: [num_queries, max_num_blocks_per_seq]
167
+ // paged_kv_indices is flattened block_tables.
168
+ for (int idx = tid; idx < (num_seqs * max_num_blocks_per_seq);
169
+ idx += (gridDim.x * blockDim.x)) {
170
+ // block_tables-row = paged_kv_indptr[queryNum]
171
+ int queryNum = idx / max_num_blocks_per_seq;
172
+ int col = idx % max_num_blocks_per_seq;
173
+ if (queryNum < num_queries && col < block_table_bound_ptr[queryNum]) {
174
+ int indices_arr_idx = paged_kv_indptr_ptr[queryNum] + col;
175
+ int block_tables_idx = queryNum * max_num_blocks_per_seq + col;
176
+ paged_kv_indices_ptr[indices_arr_idx] =
177
+ block_tables_ptr[block_tables_idx];
178
+ }
179
}
180
}
181
@@ -247,22 +263,16 @@ void advance_step_flashinfer(
263
int threads;
264
cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
265
cudaDeviceGetAttribute(&threads, cudaDevAttrMaxThreadsPerBlock, dev);
250
- if (logging) {
251
- printf("launching kernel with %d blocks\n", blocks);
252
- }
266
254
- // TODO(will): support arbitrary block_tables stride
255
- if ((blocks * threads) / block_tables.stride(0) < num_queries) {
256
- TORCH_CHECK(false,
257
- "multi-step: not enough threads to map block_table to"
258
- "FlashInfer's paged_kv_indices on GPU. Try reducing the number "
259
- "of seqs,",
260
- " increasing the block size or take smaller steps.",
261
- " num_queries = ", num_queries,
262
- " block_tables.stride(0) = ", block_tables.stride(0),
263
- " blocks = ", blocks, " max_threads = ", threads);
267
+ int block_tables_stride = block_tables.stride(0);
268
+ TORCH_CHECK((blocks * threads > num_queries),
269
+ "multi-step: not enough threads to map to num_queries = ",
270
+ num_queries, " block_tables.stride(0) = ", block_tables.stride(0),
271
+ " blocks = ", blocks, " max_threads = ", threads);
272
+ if (logging) {
273
+ printf("launching kernels with %d blocks and %d threads\n", blocks,
274
+ threads);
275
}
265
-
276
advance_step_flashinfer_kernel<<<blocks, threads, 0, stream>>>(
277
threads, num_seqs, num_queries, block_size,
278
reinterpret_cast<long*>(input_tokens.data_ptr()),
@@ -281,7 +291,7 @@ void advance_step_flashinfer(
291
reinterpret_cast<int*>(block_table_bound.data_ptr()));
292
293
advance_step_flashinfer_indices_kernel<<<blocks, threads, 0, stream>>>(
284
- threads, num_seqs, num_queries,
294
+ num_seqs, num_queries,
295
reinterpret_cast<int const*>(block_tables.data_ptr()),
296
block_tables.stride(0),
297
reinterpret_cast<int*>(paged_kv_indices.data_ptr()),