A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
add an env var for path to pre-downloaded flashinfer cubin files (#22675)
Shiyan Deng committed
Aug 22, 2025 at 12:25 UTC
da65bec3096b704f70606e9d11ee18aae453bbae
2 files changed
+11
vllm/envs.py
+6
@@ -158,6 +158,7 @@ if TYPE_CHECKING:
158
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = False
159
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
160
VLLM_USE_TRTLLM_ATTENTION: Optional[str] = None
161
+ VLLM_HAS_FLASHINFER_CUBIN: bool = False
162
VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False
163
VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
164
VLLM_TUNED_CONFIG_FOLDER: Optional[str] = None
@@ -1105,6 +1106,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
1106
"VLLM_USE_TRTLLM_ATTENTION":
1107
lambda: os.getenv("VLLM_USE_TRTLLM_ATTENTION", None),
1108
1109
+ # If set, it means we pre-downloaded cubin files and flashinfer will
1110
+ # read the cubin files directly.
1111
+ "VLLM_HAS_FLASHINFER_CUBIN":
1112
+ lambda: os.getenv("VLLM_HAS_FLASHINFER_CUBIN", False),
1113
+
1114
# If set to 1, force the use of TRTLLM FP4 GEMM backend in flashinfer.
1115
# Otherwise, uses the first available of: flashinfer cutlass GEMM,
1116
# vllm cutlass GEMM, marlin GEMM.
vllm/utils/flashinfer.py
+5
@@ -132,6 +132,11 @@ def has_nvidia_artifactory() -> bool:
132
This checks connectivity to the kernel inference library artifactory
133
which is required for downloading certain cubin kernels like TRTLLM FHMA.
134
"""
135
+ # Since FLASHINFER_CUBIN_DIR defines the pre-downloaded cubins path, when
136
+ # it's true, we could assume the cubins are available.
137
+ if envs.VLLM_HAS_FLASHINFER_CUBIN:
138
+ return True
139
+
140
try:
141
# Use a short timeout to avoid blocking for too long
142
response = requests.get(FLASHINFER_CUBINS_REPOSITORY, timeout=5)