@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c9a5258f9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Kernel] Add env variable to force flashinfer backend to enable tensor cores (#9497)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Chih-Chieh Yang <chih.chieh.yang@ibm.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Thomas Parnell committed Oct 19, 2024 at 02:55 UTC 0c9a5258f905ff3b03019f9134914ab90dbdac01
2 files changed +11 -2
vllm/attention/backends/flashinfer.py
+5 -2
@@ -17,6 +17,7 @@ except ImportError:
17
18 import torch
19
20 +import vllm.envs as envs
21 from vllm import _custom_ops as ops
22 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
23 AttentionMetadata,
@@ -124,7 +125,8 @@ class FlashInferState(AttentionState):
125 self.runner.parallel_config))
126 num_kv_heads = self.runner.model_config.get_num_kv_heads(
127 self.runner.parallel_config)
127 - use_tensor_cores = num_qo_heads // num_kv_heads > 4
128 + use_tensor_cores = envs.VLLM_FLASHINFER_FORCE_TENSOR_CORES or (
129 + num_qo_heads // num_kv_heads > 4)
130 self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
131 self._get_workspace_buffer(),
132 "NHD",
@@ -183,7 +185,8 @@ class FlashInferState(AttentionState):
185 self.runner.parallel_config))
186 num_kv_heads = self.runner.model_config.get_num_kv_heads(
187 self.runner.parallel_config)
186 - use_tensor_cores = num_qo_heads // num_kv_heads > 4
188 + use_tensor_cores = envs.VLLM_FLASHINFER_FORCE_TENSOR_CORES or (
189 + num_qo_heads // num_kv_heads > 4)
190 self._graph_decode_wrapper = \
191 CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
192 self._graph_decode_workspace_buffer, _indptr_buffer,
vllm/envs.py
+6
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
32 VLLM_ATTENTION_BACKEND: Optional[str] = None
33 VLLM_USE_FLASHINFER_SAMPLER: bool = False
34 VLLM_USE_FLASHINFER_REJECTION_SAMPLER: bool = False
35 + VLLM_FLASHINFER_FORCE_TENSOR_CORES: bool = False
36 VLLM_PP_LAYER_PARTITION: Optional[str] = None
37 VLLM_CPU_KVCACHE_SPACE: int = 0
38 VLLM_CPU_OMP_THREADS_BIND: str = ""
@@ -286,6 +287,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
287 "VLLM_USE_FLASHINFER_SAMPLER":
288 lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_SAMPLER", "0"))),
289
290 + # If set, vllm will force flashinfer to use tensor cores;
291 + # otherwise will use heuristic based on model architecture.
292 + "VLLM_FLASHINFER_FORCE_TENSOR_CORES":
293 + lambda: bool(int(os.getenv("VLLM_FLASHINFER_FORCE_TENSOR_CORES", "0"))),
294 +
295 # Pipeline stage partition strategy
296 "VLLM_PP_LAYER_PARTITION":
297 lambda: os.getenv("VLLM_PP_LAYER_PARTITION", None),