A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Doc] add env docs for flashinfer backend (#6437)
DefTruth committed
Jul 15, 2024 at 12:16 UTC
44874a0bf970ae55c487a1dc09b25bd308872f7c
1 file changed
+1
vllm/envs.py
+1
@@ -206,6 +206,7 @@ environment_variables: Dict[str, Callable[[], Any]] = {
206
# - "FLASH_ATTN": use FlashAttention
207
# - "XFORMERS": use XFormers
208
# - "ROCM_FLASH": use ROCmFlashAttention
209
+ # - "FLASHINFER": use flashinfer
210
"VLLM_ATTENTION_BACKEND":
211
lambda: os.getenv("VLLM_ATTENTION_BACKEND", None),
212