A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
Revert #7509 (#7887)
Cody Yu committed
Aug 27, 2024 at 00:16 UTC
9606c7197df073e373ab9e716a62dd4c35398865
1 file changed
+2
-4
vllm/attention/backends/flashinfer.py
+2
-4
@@ -113,8 +113,7 @@ class FlashInferState(AttentionState):
113
self.runner.parallel_config))
114
num_kv_heads = self.runner.model_config.get_num_kv_heads(
115
self.runner.parallel_config)
116
- use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
117
- (1, 2, 4, 8)
116
+ use_tensor_cores = num_qo_heads // num_kv_heads > 4
117
self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
118
self._get_workspace_buffer(),
119
"NHD",
@@ -172,8 +171,7 @@ class FlashInferState(AttentionState):
171
self.runner.parallel_config))
172
num_kv_heads = self.runner.model_config.get_num_kv_heads(
173
self.runner.parallel_config)
175
- use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
176
- (1, 2, 4, 8)
174
+ use_tensor_cores = num_qo_heads // num_kv_heads > 4
175
self._graph_decode_wrapper = \
176
CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
177
self._graph_decode_workspace_buffer, _indptr_buffer,