@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b6dde33019
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Core] Flashinfer - Remove advance step size restriction (#10282)

Pavani Majety committed Nov 13, 2024 at 00:29 UTC b6dde330198848a4a9903c1f0f97c3235fba0ba0
1 file changed +38 -28
csrc/prepare_inputs/advance_step.cu
+38 -28
@@ -88,6 +88,7 @@ inline void verify_tensor(std::string const& name, torch::Tensor const& t,
88 }
89 }
90
91 +/// each thread processes a block per query
92 __global__ void advance_step_flashinfer_kernel(
93 int num_threads, int num_seqs, int num_queries, int block_size,
94 long* input_tokens_ptr, long const* sampled_token_ids_ptr,
@@ -134,8 +135,10 @@ __global__ void advance_step_flashinfer_indptr_kernel(
135 int num_threads, int num_seqs, int num_queries, int* paged_kv_indptr_ptr,
136 int* block_table_bound_ptr) {
137 int idx = blockIdx.x * num_threads + threadIdx.x;
137 -
138 // Update paged_kv_indptr
139 + if (idx == 0) {
140 + paged_kv_indptr_ptr[idx] = 0;
141 + }
142 if (idx < num_queries) {
143 int sum = 0;
144 for (int i = 0; i <= idx; ++i) {
@@ -146,20 +149,33 @@ __global__ void advance_step_flashinfer_indptr_kernel(
149 }
150
151 __global__ void advance_step_flashinfer_indices_kernel(
149 - int num_threads, int num_seqs, int num_queries, int const* block_tables_ptr,
150 - int64_t const block_tables_stride, int* paged_kv_indices_ptr,
152 + int num_seqs, int num_queries, int const* block_tables_ptr,
153 + int64_t const max_num_blocks_per_seq, int* paged_kv_indices_ptr,
154 int* paged_kv_indptr_ptr, int* block_table_bound_ptr) {
152 - int idx = blockIdx.x * num_threads + threadIdx.x;
153 - int row = idx / block_tables_stride;
154 - int col = idx % block_tables_stride;
155 -
156 - if (row < num_queries && col < block_table_bound_ptr[row]) {
157 - paged_kv_indices_ptr[paged_kv_indptr_ptr[row] + col] =
158 - block_tables_ptr[row * block_tables_stride + col];
155 + // note: max_num_blocks_per_seq = block_tables.stride(0)
156 + int tid = blockIdx.x * blockDim.x + threadIdx.x;
157 +
158 + // when cuda graphs are enabled, paged_kv_indptr tensor
159 + // has to be updated for the padded queries
160 + // tid represents a query# for paged_kv_indptr tensor
161 + if (num_queries < tid && tid <= num_seqs) {
162 + paged_kv_indptr_ptr[tid] = paged_kv_indptr_ptr[num_queries];
163 }
160 - // if cudagraph, fill padded seqs with the last valid seq's indptr
161 - if (num_queries < row && row <= num_seqs) {
162 - paged_kv_indptr_ptr[row] = paged_kv_indptr_ptr[num_queries];
164 +
165 + // each thread processes a block_ptr in block_tables
166 + // block_tables shape: [num_queries, max_num_blocks_per_seq]
167 + // paged_kv_indices is flattened block_tables.
168 + for (int idx = tid; idx < (num_seqs * max_num_blocks_per_seq);
169 + idx += (gridDim.x * blockDim.x)) {
170 + // block_tables-row = paged_kv_indptr[queryNum]
171 + int queryNum = idx / max_num_blocks_per_seq;
172 + int col = idx % max_num_blocks_per_seq;
173 + if (queryNum < num_queries && col < block_table_bound_ptr[queryNum]) {
174 + int indices_arr_idx = paged_kv_indptr_ptr[queryNum] + col;
175 + int block_tables_idx = queryNum * max_num_blocks_per_seq + col;
176 + paged_kv_indices_ptr[indices_arr_idx] =
177 + block_tables_ptr[block_tables_idx];
178 + }
179 }
180 }
181
@@ -247,22 +263,16 @@ void advance_step_flashinfer(
263 int threads;
264 cudaDeviceGetAttribute(&blocks, cudaDevAttrMultiProcessorCount, dev);
265 cudaDeviceGetAttribute(&threads, cudaDevAttrMaxThreadsPerBlock, dev);
250 - if (logging) {
251 - printf("launching kernel with %d blocks\n", blocks);
252 - }
266
254 - // TODO(will): support arbitrary block_tables stride
255 - if ((blocks * threads) / block_tables.stride(0) < num_queries) {
256 - TORCH_CHECK(false,
257 - "multi-step: not enough threads to map block_table to"
258 - "FlashInfer's paged_kv_indices on GPU. Try reducing the number "
259 - "of seqs,",
260 - " increasing the block size or take smaller steps.",
261 - " num_queries = ", num_queries,
262 - " block_tables.stride(0) = ", block_tables.stride(0),
263 - " blocks = ", blocks, " max_threads = ", threads);
267 + int block_tables_stride = block_tables.stride(0);
268 + TORCH_CHECK((blocks * threads > num_queries),
269 + "multi-step: not enough threads to map to num_queries = ",
270 + num_queries, " block_tables.stride(0) = ", block_tables.stride(0),
271 + " blocks = ", blocks, " max_threads = ", threads);
272 + if (logging) {
273 + printf("launching kernels with %d blocks and %d threads\n", blocks,
274 + threads);
275 }
265 -
276 advance_step_flashinfer_kernel<<<blocks, threads, 0, stream>>>(
277 threads, num_seqs, num_queries, block_size,
278 reinterpret_cast<long*>(input_tokens.data_ptr()),
@@ -281,7 +291,7 @@ void advance_step_flashinfer(
291 reinterpret_cast<int*>(block_table_bound.data_ptr()));
292
293 advance_step_flashinfer_indices_kernel<<<blocks, threads, 0, stream>>>(
284 - threads, num_seqs, num_queries,
294 + num_seqs, num_queries,
295 reinterpret_cast<int const*>(block_tables.data_ptr()),
296 block_tables.stride(0),
297 reinterpret_cast<int*>(paged_kv_indices.data_ptr()),