@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6e4a130b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Minor] Remove gather_cached_kv kernel (#3043)

Woosuk Kwon committed Feb 26, 2024 at 15:00 UTC d6e4a130b028f42a7f413d99eb91a4395fa7a04a
3 files changed -172
csrc/cache.h
-7
@@ -23,13 +23,6 @@ void reshape_and_cache(
23 torch::Tensor& slot_mapping,
24 const std::string& kv_cache_dtype);
25
26 -void gather_cached_kv(
27 - torch::Tensor& key,
28 - torch::Tensor& value,
29 - torch::Tensor& key_cache,
30 - torch::Tensor& value_cache,
31 - torch::Tensor& slot_mapping);
32 -
26 // Just for unittest
27 void convert_fp8_e5m2(
28 torch::Tensor& src_cache,
csrc/cache_kernels.cu
-161
@@ -269,167 +269,6 @@ void reshape_and_cache(
269
270 namespace vllm {
271
272 -// Grid: (num_blocks, block_size).
273 -template<typename scalar_t>
274 -__global__ void gather_cached_kv_kernel(
275 - scalar_t* __restrict__ key, // [num_tokens, [stride], num_heads, head_size]
276 - scalar_t* __restrict__ value, // [num_tokens, [stride], num_heads, head_size]
277 - const scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
278 - const scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
279 - const int* __restrict__ slot_mapping, // [num_tokens]
280 - const int key_stride,
281 - const int value_stride,
282 - const int num_heads,
283 - const int head_size,
284 - const int block_size,
285 - const int x) {
286 - const int token_idx = blockIdx.x;
287 - const int slot_idx = slot_mapping[token_idx];
288 - const int block_idx = slot_idx / block_size;
289 - const int block_offset = slot_idx % block_size;
290 -
291 - const int num_tokens = num_heads * head_size;
292 - for (int i = threadIdx.x; i < num_tokens; i += blockDim.x) {
293 - const int tgt_key_idx = token_idx * key_stride + i;
294 - const int tgt_value_idx = token_idx * value_stride + i;
295 -
296 - const int head_idx = i / head_size;
297 - const int head_offset = i % head_size;
298 - const int x_idx = head_offset / x; // the offset of the [head_size/x] dimension
299 - const int x_offset = head_offset % x;
300 -
301 - const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
302 - + head_idx * (head_size / x) * block_size * x
303 - + x_idx * block_size * x
304 - + block_offset * x
305 - + x_offset;
306 - const int src_value_idx = block_idx * num_heads * head_size * block_size
307 - + head_idx * head_size * block_size
308 - + head_offset * block_size
309 - + block_offset;
310 -
311 - key[tgt_key_idx] = VLLM_LDG(&key_cache[src_key_idx]);
312 - value[tgt_value_idx] = VLLM_LDG(&value_cache[src_value_idx]);
313 - }
314 -}
315 -
316 -template <typename scalar_t>
317 -__global__ void gather_cached_kv_kernel_optimized(
318 - scalar_t *__restrict__ key, // [num_tokens, [stride], num_heads, head_size]
319 - scalar_t *__restrict__ value, // [num_tokens, [stride], num_heads, head_size]
320 - const scalar_t *__restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
321 - const scalar_t *__restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
322 - const int *__restrict__ slot_mapping, // [num_tokens]
323 - const int key_stride,
324 - const int value_stride,
325 - const int num_heads,
326 - const int head_size,
327 - const int block_size,
328 - const int x)
329 -{
330 - const int token_idx = blockIdx.x;
331 - const int slot_idx = slot_mapping[token_idx];
332 - const int block_idx = slot_idx / block_size;
333 - const int block_offset = slot_idx % block_size;
334 -
335 - const int dim = num_heads * head_size;
336 - assert(dim % 4 == 0); // this is true for known use cases
337 - const int unroll_factor = 4;
338 - const int unrolled_dim = dim / unroll_factor;
339 -
340 - for (int i = threadIdx.x; i < unrolled_dim; i += blockDim.x)
341 - {
342 - int tgt_key_indices[unroll_factor];
343 - int tgt_value_indices[unroll_factor];
344 - int src_key_indices[unroll_factor];
345 - int src_value_indices[unroll_factor];
346 - scalar_t keys_to_store[unroll_factor];
347 - scalar_t values_to_store[unroll_factor];
348 -
349 - #pragma unroll
350 - for (int j = 0; j < unroll_factor; ++j)
351 - {
352 - int index = i + j * unrolled_dim;
353 -
354 - const int tgt_key_idx = token_idx * key_stride + index;
355 - const int tgt_value_idx = token_idx * value_stride + index;
356 -
357 - const int head_idx = index / head_size;
358 - const int head_offset = index % head_size;
359 - const int x_idx = head_offset / x;
360 - const int x_offset = head_offset % x;
361 -
362 - const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
363 - + head_idx * (head_size / x) * block_size * x
364 - + x_idx * block_size * x
365 - + block_offset * x
366 - + x_offset;
367 - const int src_value_idx = block_idx * num_heads * head_size * block_size
368 - + head_idx * head_size * block_size
369 - + head_offset * block_size
370 - + block_offset;
371 -
372 - tgt_key_indices[j] = tgt_key_idx;
373 - tgt_value_indices[j] = tgt_value_idx;
374 - src_key_indices[j] = src_key_idx;
375 - src_value_indices[j] = src_value_idx;
376 -
377 - keys_to_store[j] = VLLM_LDG(&key_cache[src_key_idx]);
378 - values_to_store[j] = VLLM_LDG(&value_cache[src_value_idx]);
379 - }
380 -
381 - #pragma unroll
382 - for (int j = 0; j < unroll_factor; ++j)
383 - {
384 - key[tgt_key_indices[j]] = keys_to_store[j];
385 - value[tgt_value_indices[j]] = values_to_store[j];
386 - }
387 - }
388 -}
389 -
390 -} // namespace vllm
391 -
392 -void gather_cached_kv(
393 - torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
394 - torch::Tensor& value, // [out] [num_tokens, num_heads, head_size]
395 - torch::Tensor& key_cache, // [in] [num_blocks, num_heads, head_size/x, block_size, x]
396 - torch::Tensor& value_cache, // [in] [num_blocks, num_heads, head_size, block_size]
397 - torch::Tensor& slot_mapping) // [in] [num_tokens]
398 -{
399 - int num_tokens = key.size(0);
400 - int num_heads = key.size(1);
401 - int head_size = key.size(2);
402 - int block_size = key_cache.size(3);
403 - int x = key_cache.size(4);
404 -
405 - int key_stride = key.stride(0);
406 - int value_stride = value.stride(0);
407 -
408 - dim3 grid(num_tokens);
409 - dim3 block(std::min(num_heads * head_size, 512));
410 - const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
411 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
412 - VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
413 - key.scalar_type(),
414 - "gather_cached_kv_kernel_optimized",
415 - [&] {
416 - vllm::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
417 - key.data_ptr<scalar_t>(),
418 - value.data_ptr<scalar_t>(),
419 - key_cache.data_ptr<scalar_t>(),
420 - value_cache.data_ptr<scalar_t>(),
421 - slot_mapping.data_ptr<int>(),
422 - key_stride,
423 - value_stride,
424 - num_heads,
425 - head_size,
426 - block_size,
427 - x);
428 - });
429 -}
430 -
431 -namespace vllm {
432 -
272 template<typename Tout, typename Tin>
273 __global__ void convert_fp8_e5m2_kernel(
274 const Tin* __restrict__ src_cache,
csrc/pybind.cpp
-4
@@ -79,10 +79,6 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
79 "reshape_and_cache",
80 &reshape_and_cache,
81 "Reshape the key and value tensors and cache them");
82 - cache_ops.def(
83 - "gather_cached_kv",
84 - &gather_cached_kv,
85 - "Gather key and value from the cache into contiguous QKV tensors");
82 cache_ops.def(
83 "convert_fp8_e5m2",
84 &convert_fp8_e5m2,