@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9606c7197d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

Revert #7509 (#7887)

Cody Yu committed Aug 27, 2024 at 00:16 UTC 9606c7197df073e373ab9e716a62dd4c35398865
1 file changed +2 -4
vllm/attention/backends/flashinfer.py
+2 -4
@@ -113,8 +113,7 @@ class FlashInferState(AttentionState):
113 self.runner.parallel_config))
114 num_kv_heads = self.runner.model_config.get_num_kv_heads(
115 self.runner.parallel_config)
116 - use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
117 - (1, 2, 4, 8)
116 + use_tensor_cores = num_qo_heads // num_kv_heads > 4
117 self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
118 self._get_workspace_buffer(),
119 "NHD",
@@ -172,8 +171,7 @@ class FlashInferState(AttentionState):
171 self.runner.parallel_config))
172 num_kv_heads = self.runner.model_config.get_num_kv_heads(
173 self.runner.parallel_config)
175 - use_tensor_cores = (num_qo_heads // num_kv_heads) not in \
176 - (1, 2, 4, 8)
174 + use_tensor_cores = num_qo_heads // num_kv_heads > 4
175 self._graph_decode_wrapper = \
176 CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
177 self._graph_decode_workspace_buffer, _indptr_buffer,