A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[BUG] fix crash on flashinfer backend with cudagraph disabled, when attention group_size not in [1,2,4,8] (#7509)
LI MOU committed
Aug 21, 2024 at 23:54 UTC
53328d7536b3e8ea4863e351b09e4284f5601cae
2 files changed
+9
-4
tests/kernels/test_flashinfer.py
+5
-2
@@ -4,7 +4,7 @@ import flashinfer
4
import pytest
5
import torch
6
7
-NUM_HEADS = [(16, 16), (32, 8), (64, 8)]
7
+NUM_HEADS = [(16, 16), (32, 8), (64, 8), (6, 1)]
8
HEAD_SIZES = [128, 256]
9
BLOCK_SIZES = [16, 32]
10
DTYPES = [torch.float16, torch.bfloat16]
@@ -123,7 +123,10 @@ def test_flashinfer_decode_with_paged_kv(kv_lens: List[int],
123
124
workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8)
125
wrapper = flashinfer.\
126
- BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD")
126
+ BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD",
127
+ use_tensor_cores=(
128
+ (num_query_heads//num_kv_heads) not in (1, 2, 4, 8))
129
+ )
130
wrapper.begin_forward(kv_indptr,
131
kv_indices,
132
kv_last_page_lens,
vllm/attention/backends/flashinfer.py
+4
-2
@@ -113,7 +113,8 @@ class FlashInferState(AttentionState):
113
self.runner.parallel_config))
114
num_kv_heads = self.runner.model_config.get_num_kv_heads(
115
self.runner.parallel_config)
116
- use_tensor_cores = num_qo_heads // num_kv_heads >= 4
116
+ use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
117
+ (1, 2, 4, 8)
118
self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
119
self._get_workspace_buffer(),
120
"NHD",
@@ -171,7 +172,8 @@ class FlashInferState(AttentionState):
172
self.runner.parallel_config))
173
num_kv_heads = self.runner.model_config.get_num_kv_heads(
174
self.runner.parallel_config)
174
- use_tensor_cores = num_qo_heads // num_kv_heads >= 4
175
+ use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
176
+ (1, 2, 4, 8)
177
self._graph_decode_wrapper = \
178
CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
179
self._graph_decode_workspace_buffer, _indptr_buffer,