@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 53328d7536
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[BUG] fix crash on flashinfer backend with cudagraph disabled, when attention group_size not in [1,2,4,8] (#7509)

LI MOU committed Aug 21, 2024 at 23:54 UTC 53328d7536b3e8ea4863e351b09e4284f5601cae
2 files changed +9 -4
tests/kernels/test_flashinfer.py
+5 -2
@@ -4,7 +4,7 @@ import flashinfer
4 import pytest
5 import torch
6
7 -NUM_HEADS = [(16, 16), (32, 8), (64, 8)]
7 +NUM_HEADS = [(16, 16), (32, 8), (64, 8), (6, 1)]
8 HEAD_SIZES = [128, 256]
9 BLOCK_SIZES = [16, 32]
10 DTYPES = [torch.float16, torch.bfloat16]
@@ -123,7 +123,10 @@ def test_flashinfer_decode_with_paged_kv(kv_lens: List[int],
123
124 workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
125 wrapper = flashinfer.\
126 - BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD")
126 + BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD",
127 + use_tensor_cores=(
128 + (num_query_heads//num_kv_heads) not in (1, 2, 4, 8))
129 + )
130 wrapper.begin_forward(kv_indptr,
131 kv_indices,
132 kv_last_page_lens,
vllm/attention/backends/flashinfer.py
+4 -2
@@ -113,7 +113,8 @@ class FlashInferState(AttentionState):
113 self.runner.parallel_config))
114 num_kv_heads = self.runner.model_config.get_num_kv_heads(
115 self.runner.parallel_config)
116 - use_tensor_cores = num_qo_heads // num_kv_heads >= 4
116 + use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
117 + (1, 2, 4, 8)
118 self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
119 self._get_workspace_buffer(),
120 "NHD",
@@ -171,7 +172,8 @@ class FlashInferState(AttentionState):
172 self.runner.parallel_config))
173 num_kv_heads = self.runner.model_config.get_num_kv_heads(
174 self.runner.parallel_config)
174 - use_tensor_cores = num_qo_heads // num_kv_heads >= 4
175 + use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
176 + (1, 2, 4, 8)
177 self._graph_decode_wrapper = \
178 CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
179 self._graph_decode_workspace_buffer, _indptr_buffer,