A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Kernel] Add env variable to force flashinfer backend to enable tensor cores (#9497)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Chih-Chieh Yang <chih.chieh.yang@ibm.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Thomas Parnell committed
Oct 19, 2024 at 02:55 UTC
0c9a5258f905ff3b03019f9134914ab90dbdac01
2 files changed
+11
-2
vllm/attention/backends/flashinfer.py
+5
-2
@@ -17,6 +17,7 @@ except ImportError:
17
18
import torch
19
20
+import vllm.envs as envs
21
from vllm import _custom_ops as ops
22
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
23
AttentionMetadata,
@@ -124,7 +125,8 @@ class FlashInferState(AttentionState):
125
self.runner.parallel_config))
126
num_kv_heads = self.runner.model_config.get_num_kv_heads(
127
self.runner.parallel_config)
127
- use_tensor_cores = num_qo_heads // num_kv_heads > 4
128
+ use_tensor_cores = envs.VLLM_FLASHINFER_FORCE_TENSOR_CORES or (
129
+ num_qo_heads // num_kv_heads > 4)
130
self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
131
self._get_workspace_buffer(),
132
"NHD",
@@ -183,7 +185,8 @@ class FlashInferState(AttentionState):
185
self.runner.parallel_config))
186
num_kv_heads = self.runner.model_config.get_num_kv_heads(
187
self.runner.parallel_config)
186
- use_tensor_cores = num_qo_heads // num_kv_heads > 4
188
+ use_tensor_cores = envs.VLLM_FLASHINFER_FORCE_TENSOR_CORES or (
189
+ num_qo_heads // num_kv_heads > 4)
190
self._graph_decode_wrapper = \
191
CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
192
self._graph_decode_workspace_buffer, _indptr_buffer,
vllm/envs.py
+6
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
32
VLLM_ATTENTION_BACKEND: Optional[str] = None
33
VLLM_USE_FLASHINFER_SAMPLER: bool = False
34
VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False
35
+ VLLM_FLASHINFER_FORCE_TENSOR_CORES: bool = False
36
VLLM_PP_LAYER_PARTITION: Optional[str] = None
37
VLLM_CPU_KVCACHE_SPACE: int = 0
38
VLLM_CPU_OMP_THREADS_BIND: str = ""
@@ -286,6 +287,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
287
"VLLM_USE_FLASHINFER_SAMPLER":
288
lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))),
289
290
+ # If set, vllm will force flashinfer to use tensor cores;
291
+ # otherwise will use heuristic based on model architecture.
292
+ "VLLM_FLASHINFER_FORCE_TENSOR_CORES":
293
+ lambda: bool(int(os.getenv("VLLM_FLASHINFER_FORCE_TENSOR_CORES", "0"))),
294
+
295
# Pipeline stage partition strategy
296
"VLLM_PP_LAYER_PARTITION":
297
lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),