@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0e63494cf3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

Add fp8 support to `reshape_and_cache_flash` (#6667)

Antoni Baum committed Jul 24, 2024 at 11:36 UTC 0e63494cf334497148ee4203b59e34b2dd53e50e
8 files changed +98 -43
csrc/cache.h
+2 -1
@@ -25,7 +25,8 @@ void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
25 torch::Tensor& key_cache,
26 torch::Tensor& value_cache,
27 torch::Tensor& slot_mapping,
28 - const std::string& kv_cache_dtype);
28 + const std::string& kv_cache_dtype,
29 + const double k_scale, const double v_scale);
30
31 // Just for unittest
32 void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,
csrc/cache_kernels.cu
+45 -30
@@ -203,17 +203,18 @@ __global__ void reshape_and_cache_kernel(
203 }
204 }
205
206 -template <typename scalar_t>
206 +template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
207 __global__ void reshape_and_cache_flash_kernel(
208 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
209 const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
210 - scalar_t* __restrict__ k_cache, // [num_blocks, block_size, num_heads,
210 + cache_t* __restrict__ key_cache, // [num_blocks, block_size, num_heads,
211 // head_size]
212 - scalar_t* __restrict__ v_cache, // [num_blocks, block_size, num_heads,
212 + cache_t* __restrict__ value_cache, // [num_blocks, block_size, num_heads,
213 // head_size]
214 const int64_t* __restrict__ slot_mapping, // [num_tokens]
215 const int block_stride, const int key_stride, const int value_stride,
216 - const int num_heads, const int head_size, const int block_size) {
216 + const int num_heads, const int head_size, const int block_size,
217 + const float k_scale, const float v_scale) {
218 const int64_t token_idx = blockIdx.x;
219 const int64_t slot_idx = slot_mapping[token_idx];
220 // NOTE: slot_idx can be -1 if the token is padded
@@ -228,11 +229,20 @@ __global__ void reshape_and_cache_flash_kernel(
229 const int64_t src_value_idx = token_idx * value_stride + i;
230 const int head_idx = i / head_size;
231 const int head_offset = i % head_size;
231 - const int64_t tgt_value_idx = block_idx * block_stride +
232 - block_offset * num_heads * head_size +
233 - head_idx * head_size + head_offset;
234 - k_cache[tgt_value_idx] = key[src_key_idx];
235 - v_cache[tgt_value_idx] = value[src_value_idx];
232 + const int64_t tgt_key_value_idx = block_idx * block_stride +
233 + block_offset * num_heads * head_size +
234 + head_idx * head_size + head_offset;
235 + scalar_t tgt_key = key[src_key_idx];
236 + scalar_t tgt_value = value[src_value_idx];
237 + if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
238 + key_cache[tgt_key_value_idx] = tgt_key;
239 + value_cache[tgt_key_value_idx] = tgt_value;
240 + } else {
241 + key_cache[tgt_key_value_idx] =
242 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_key, k_scale);
243 + value_cache[tgt_key_value_idx] =
244 + fp8::scaled_convert<cache_t, scalar_t, kv_dt>(tgt_value, v_scale);
245 + }
246 }
247 }
248 } // namespace vllm
@@ -278,40 +288,45 @@ void reshape_and_cache(
288 CALL_RESHAPE_AND_CACHE)
289 }
290
291 +// KV_T is the stored data type of kv-cache.
292 +// CACHE_T is the data type of key and value tensors.
293 +// KV_DTYPE is the real data type of kv-cache.
294 +#define CALL_RESHAPE_AND_CACHE_FLASH(KV_T, CACHE_T, KV_DTYPE) \
295 + vllm::reshape_and_cache_flash_kernel<KV_T, CACHE_T, KV_DTYPE> \
296 + <<<grid, block, 0, stream>>>( \
297 + reinterpret_cast<KV_T*>(key.data_ptr()), \
298 + reinterpret_cast<KV_T*>(value.data_ptr()), \
299 + reinterpret_cast<CACHE_T*>(key_cache.data_ptr()), \
300 + reinterpret_cast<CACHE_T*>(value_cache.data_ptr()), \
301 + slot_mapping.data_ptr<int64_t>(), block_stride, key_stride, \
302 + value_stride, num_heads, head_size, block_size, k_scale, v_scale);
303 +
304 void reshape_and_cache_flash(
282 - torch::Tensor& key, // [num_tokens, num_heads, head_size]
283 - torch::Tensor& value, // [num_tokens, num_heads, head_size]
284 - torch::Tensor& k_cache, // [num_blocks, block_size, num_heads, head_size]
285 - torch::Tensor& v_cache, // [num_blocks, block_size, num_heads, head_size]
305 + torch::Tensor& key, // [num_tokens, num_heads, head_size]
306 + torch::Tensor& value, // [num_tokens, num_heads, head_size]
307 + torch::Tensor& key_cache, // [num_blocks, block_size, num_heads, head_size]
308 + torch::Tensor&
309 + value_cache, // [num_blocks, block_size, num_heads, head_size]
310 torch::Tensor& slot_mapping, // [num_tokens]
287 - const std::string& kv_cache_dtype) {
288 - // FIXME: only support auto datatype, does not support fp8
289 - if (kv_cache_dtype != "auto") {
290 - TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
291 - }
311 + const std::string& kv_cache_dtype, const double k_scale,
312 + const double v_scale) {
313 int num_tokens = key.size(0);
314 int num_heads = key.size(1);
315 int head_size = key.size(2);
295 - int block_size = k_cache.size(1);
316 + int block_size = key_cache.size(1);
317
318 int key_stride = key.stride(0);
319 int value_stride = value.stride(0);
299 - int block_stride = k_cache.stride(0);
300 - TORCH_CHECK(k_cache.stride(0) == v_cache.stride(0));
320 + int block_stride = key_cache.stride(0);
321 + TORCH_CHECK(key_cache.stride(0) == value_cache.stride(0));
322
323 dim3 grid(num_tokens);
324 dim3 block(std::min(num_heads * head_size, 512));
325 const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
326 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
306 - VLLM_DISPATCH_FLOATING_TYPES(
307 - key.scalar_type(), "reshape_and_cache_flash", [&] {
308 - vllm::reshape_and_cache_flash_kernel<scalar_t>
309 - <<<grid, block, 0, stream>>>(
310 - key.data_ptr<scalar_t>(), value.data_ptr<scalar_t>(),
311 - k_cache.data_ptr<scalar_t>(), v_cache.data_ptr<scalar_t>(),
312 - slot_mapping.data_ptr<int64_t>(), block_stride, key_stride,
313 - value_stride, num_heads, head_size, block_size);
314 - });
327 +
328 + DISPATCH_BY_KV_CACHE_DTYPE(key.dtype(), kv_cache_dtype,
329 + CALL_RESHAPE_AND_CACHE_FLASH);
330 }
331
332 namespace vllm {
csrc/torch_bindings.cpp
+2 -1
@@ -248,7 +248,8 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
248 " Tensor! key_cache,"
249 " Tensor! value_cache,"
250 " Tensor slot_mapping,"
251 - " str kv_cache_dtype) -> ()");
251 + " str kv_cache_dtype,"
252 + " float k_scale, float v_scale) -> ()");
253 cache_ops.impl("reshape_and_cache_flash", torch::kCUDA,
254 &reshape_and_cache_flash);
255
tests/kernels/test_cache.py
+34 -8
@@ -215,8 +215,6 @@ def test_reshape_and_cache_flash(
215 device: str,
216 kv_cache_dtype: str,
217 ) -> None:
218 - if kv_cache_dtype == "fp8":
219 - pytest.skip()
218 random.seed(seed)
219 torch.random.manual_seed(seed)
220 torch.cuda.manual_seed(seed)
@@ -248,15 +246,33 @@ def test_reshape_and_cache_flash(
246 dtype,
247 device=device,
248 )
251 - key_cache, value_cache = key_caches[0], value_caches[0]
249 + key_cache, value_cache = key_caches[0].contiguous(
250 + ), value_caches[0].contiguous()
251 + del key_caches
252 + del value_caches
253
254 # Clone the KV caches.
254 - cloned_key_cache = key_cache.clone()
255 - cloned_value_cache = value_cache.clone()
255 + if kv_cache_dtype == "fp8":
256 + cloned_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
257 + ops.convert_fp8(cloned_key_cache, key_cache)
258 + cloned_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
259 + ops.convert_fp8(cloned_value_cache, value_cache)
260 + else:
261 + cloned_key_cache = key_cache.clone()
262 + cloned_value_cache = value_cache.clone()
263 +
264 + # Using default kv_scale
265 + k_scale = v_scale = 1.0
266
267 # Call the reshape_and_cache kernel.
268 ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
259 - slot_mapping, kv_cache_dtype)
269 + slot_mapping, kv_cache_dtype, k_scale, v_scale)
270 +
271 + if kv_cache_dtype == "fp8":
272 + result_key_cache = torch.empty_like(key_cache, dtype=torch.float16)
273 + ops.convert_fp8(result_key_cache, key_cache)
274 + result_value_cache = torch.empty_like(value_cache, dtype=torch.float16)
275 + ops.convert_fp8(result_value_cache, value_cache)
276
277 # Run the reference implementation.
278 block_indicies = torch.div(slot_mapping, block_size, rounding_mode="floor")
@@ -269,8 +285,18 @@ def test_reshape_and_cache_flash(
285 cloned_key_cache[block_idx, block_offset, :, :] = key[i]
286 cloned_value_cache[block_idx, block_offset, :, :] = value[i]
287
272 - assert torch.allclose(key_cache, cloned_key_cache)
273 - assert torch.allclose(value_cache, cloned_value_cache)
288 + if kv_cache_dtype == "fp8":
289 + assert torch.allclose(result_key_cache,
290 + cloned_key_cache,
291 + atol=0.001,
292 + rtol=0.1)
293 + assert torch.allclose(result_value_cache,
294 + cloned_value_cache,
295 + atol=0.001,
296 + rtol=0.1)
297 + else:
298 + assert torch.allclose(key_cache, cloned_key_cache)
299 + assert torch.allclose(value_cache, cloned_value_cache)
300
301
302 @pytest.mark.parametrize("direction", COPYING_DIRECTION)
vllm/_custom_ops.py
+4 -1
@@ -426,10 +426,13 @@ def reshape_and_cache_flash(
426 value_cache: torch.Tensor,
427 slot_mapping: torch.Tensor,
428 kv_cache_dtype: str,
429 + k_scale: float,
430 + v_scale: float,
431 ) -> None:
432 torch.ops._C_cache_ops.reshape_and_cache_flash(key, value, key_cache,
433 value_cache, slot_mapping,
432 - kv_cache_dtype)
434 + kv_cache_dtype, k_scale,
435 + v_scale)
436
437
438 def copy_blocks(key_caches: List[torch.Tensor],
vllm/attention/backends/flash_attn.py
+2
@@ -478,6 +478,8 @@ class FlashAttentionImpl(AttentionImpl):
478 value_cache,
479 attn_metadata.slot_mapping.flatten(),
480 self.kv_cache_dtype,
481 + k_scale,
482 + v_scale,
483 )
484
485 num_prefill_tokens = attn_metadata.num_prefill_tokens
vllm/attention/backends/flashinfer.py
+2
@@ -489,6 +489,8 @@ class FlashInferImpl(AttentionImpl):
489 kv_cache[:, 1],
490 attn_metadata.slot_mapping.flatten(),
491 self.kv_cache_dtype,
492 + k_scale,
493 + v_scale,
494 )
495
496 query = query.contiguous(
vllm/utils.py
+7 -2
@@ -491,7 +491,6 @@ def create_kv_caches_with_random_flash(
491 seed: int = 0,
492 device: Optional[str] = "cuda",
493 ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
494 - assert cache_dtype != "fp8"
494 torch.random.manual_seed(seed)
495 if torch.cuda.is_available():
496 torch.cuda.manual_seed(seed)
@@ -507,7 +506,13 @@ def create_kv_caches_with_random_flash(
506 key_value_cache = torch.empty(size=key_value_cache_shape,
507 dtype=torch_dtype,
508 device=device)
510 - key_value_cache.uniform_(-scale, scale)
509 + if cache_dtype in ["auto", "half", "bfloat16", "float"]:
510 + key_value_cache.uniform_(-scale, scale)
511 + elif cache_dtype == 'fp8':
512 + _generate_random_fp8(key_value_cache, -scale, scale)
513 + else:
514 + raise ValueError(
515 + f"Does not support key cache of type {cache_dtype}")
516 key_caches.append(key_value_cache[:, 0])
517 value_caches.append(key_value_cache[:, 1])
518 return key_caches, value_caches