A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Minor] Remove gather_cached_kv kernel (#3043)
Woosuk Kwon committed
Feb 26, 2024 at 15:00 UTC
d6e4a130b028f42a7f413d99eb91a4395fa7a04a
3 files changed
-172
csrc/cache.h
-7
@@ -23,13 +23,6 @@ void reshape_and_cache(
23
torch::Tensor& slot_mapping,
24
const std::string& kv_cache_dtype);
25
26
-void gather_cached_kv(
27
- torch::Tensor& key,
28
- torch::Tensor& value,
29
- torch::Tensor& key_cache,
30
- torch::Tensor& value_cache,
31
- torch::Tensor& slot_mapping);
32
-
26
// Just for unittest
27
void convert_fp8_e5m2(
28
torch::Tensor& src_cache,
csrc/cache_kernels.cu
-161
@@ -269,167 +269,6 @@ void reshape_and_cache(
269
270
namespace vllm {
271
272
-// Grid: (num_blocks, block_size).
273
-template<typename scalar_t>
274
-__global__ void gather_cached_kv_kernel(
275
- scalar_t* __restrict__ key, // [num_tokens, [stride], num_heads, head_size]
276
- scalar_t* __restrict__ value, // [num_tokens, [stride], num_heads, head_size]
277
- const scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
278
- const scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
279
- const int* __restrict__ slot_mapping, // [num_tokens]
280
- const int key_stride,
281
- const int value_stride,
282
- const int num_heads,
283
- const int head_size,
284
- const int block_size,
285
- const int x) {
286
- const int token_idx = blockIdx.x;
287
- const int slot_idx = slot_mapping[token_idx];
288
- const int block_idx = slot_idx / block_size;
289
- const int block_offset = slot_idx % block_size;
290
-
291
- const int num_tokens = num_heads * head_size;
292
- for (int i = threadIdx.x; i < num_tokens; i += blockDim.x) {
293
- const int tgt_key_idx = token_idx * key_stride + i;
294
- const int tgt_value_idx = token_idx * value_stride + i;
295
-
296
- const int head_idx = i / head_size;
297
- const int head_offset = i % head_size;
298
- const int x_idx = head_offset / x; // the offset of the [head_size/x] dimension
299
- const int x_offset = head_offset % x;
300
-
301
- const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
302
- + head_idx * (head_size / x) * block_size * x
303
- + x_idx * block_size * x
304
- + block_offset * x
305
- + x_offset;
306
- const int src_value_idx = block_idx * num_heads * head_size * block_size
307
- + head_idx * head_size * block_size
308
- + head_offset * block_size
309
- + block_offset;
310
-
311
- key[tgt_key_idx] = VLLM_LDG(&key_cache[src_key_idx]);
312
- value[tgt_value_idx] = VLLM_LDG(&value_cache[src_value_idx]);
313
- }
314
-}
315
-
316
-template <typename scalar_t>
317
-__global__ void gather_cached_kv_kernel_optimized(
318
- scalar_t *__restrict__ key, // [num_tokens, [stride], num_heads, head_size]
319
- scalar_t *__restrict__ value, // [num_tokens, [stride], num_heads, head_size]
320
- const scalar_t *__restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
321
- const scalar_t *__restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
322
- const int *__restrict__ slot_mapping, // [num_tokens]
323
- const int key_stride,
324
- const int value_stride,
325
- const int num_heads,
326
- const int head_size,
327
- const int block_size,
328
- const int x)
329
-{
330
- const int token_idx = blockIdx.x;
331
- const int slot_idx = slot_mapping[token_idx];
332
- const int block_idx = slot_idx / block_size;
333
- const int block_offset = slot_idx % block_size;
334
-
335
- const int dim = num_heads * head_size;
336
- assert(dim % 4 == 0); // this is true for known use cases
337
- const int unroll_factor = 4;
338
- const int unrolled_dim = dim / unroll_factor;
339
-
340
- for (int i = threadIdx.x; i < unrolled_dim; i += blockDim.x)
341
- {
342
- int tgt_key_indices[unroll_factor];
343
- int tgt_value_indices[unroll_factor];
344
- int src_key_indices[unroll_factor];
345
- int src_value_indices[unroll_factor];
346
- scalar_t keys_to_store[unroll_factor];
347
- scalar_t values_to_store[unroll_factor];
348
-
349
- #pragma unroll
350
- for (int j = 0; j < unroll_factor; ++j)
351
- {
352
- int index = i + j * unrolled_dim;
353
-
354
- const int tgt_key_idx = token_idx * key_stride + index;
355
- const int tgt_value_idx = token_idx * value_stride + index;
356
-
357
- const int head_idx = index / head_size;
358
- const int head_offset = index % head_size;
359
- const int x_idx = head_offset / x;
360
- const int x_offset = head_offset % x;
361
-
362
- const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
363
- + head_idx * (head_size / x) * block_size * x
364
- + x_idx * block_size * x
365
- + block_offset * x
366
- + x_offset;
367
- const int src_value_idx = block_idx * num_heads * head_size * block_size
368
- + head_idx * head_size * block_size
369
- + head_offset * block_size
370
- + block_offset;
371
-
372
- tgt_key_indices[j] = tgt_key_idx;
373
- tgt_value_indices[j] = tgt_value_idx;
374
- src_key_indices[j] = src_key_idx;
375
- src_value_indices[j] = src_value_idx;
376
-
377
- keys_to_store[j] = VLLM_LDG(&key_cache[src_key_idx]);
378
- values_to_store[j] = VLLM_LDG(&value_cache[src_value_idx]);
379
- }
380
-
381
- #pragma unroll
382
- for (int j = 0; j < unroll_factor; ++j)
383
- {
384
- key[tgt_key_indices[j]] = keys_to_store[j];
385
- value[tgt_value_indices[j]] = values_to_store[j];
386
- }
387
- }
388
-}
389
-
390
-} // namespace vllm
391
-
392
-void gather_cached_kv(
393
- torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
394
- torch::Tensor& value, // [out] [num_tokens, num_heads, head_size]
395
- torch::Tensor& key_cache, // [in] [num_blocks, num_heads, head_size/x, block_size, x]
396
- torch::Tensor& value_cache, // [in] [num_blocks, num_heads, head_size, block_size]
397
- torch::Tensor& slot_mapping) // [in] [num_tokens]
398
-{
399
- int num_tokens = key.size(0);
400
- int num_heads = key.size(1);
401
- int head_size = key.size(2);
402
- int block_size = key_cache.size(3);
403
- int x = key_cache.size(4);
404
-
405
- int key_stride = key.stride(0);
406
- int value_stride = value.stride(0);
407
-
408
- dim3 grid(num_tokens);
409
- dim3 block(std::min(num_heads * head_size, 512));
410
- const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
411
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
412
- VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
413
- key.scalar_type(),
414
- "gather_cached_kv_kernel_optimized",
415
- [&] {
416
- vllm::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
417
- key.data_ptr<scalar_t>(),
418
- value.data_ptr<scalar_t>(),
419
- key_cache.data_ptr<scalar_t>(),
420
- value_cache.data_ptr<scalar_t>(),
421
- slot_mapping.data_ptr<int>(),
422
- key_stride,
423
- value_stride,
424
- num_heads,
425
- head_size,
426
- block_size,
427
- x);
428
- });
429
-}
430
-
431
-namespace vllm {
432
-
272
template<typename Tout, typename Tin>
273
__global__ void convert_fp8_e5m2_kernel(
274
const Tin* __restrict__ src_cache,
csrc/pybind.cpp
-4
@@ -79,10 +79,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
79
"reshape_and_cache",
80
&reshape_and_cache,
81
"Reshape the key and value tensors and cache them");
82
- cache_ops.def(
83
- "gather_cached_kv",
84
- &gather_cached_kv,
85
- "Gather key and value from the cache into contiguous QKV tensors");
82
cache_ops.def(
83
"convert_fp8_e5m2",
84
&convert_fp8_e5m2,