A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 06:45 UTC
[CI failure] Temporarily disable using persistent cache for flashinfer autotune (#43119)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Signed-off-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Wei Zhao committed
May 19, 2026 at 14:44 UTC
9aaf83ef502fc37bc647f6e474314d48ba36cd1c
1 file changed
+16
vllm/model_executor/warmup/kernel_warmup.py
+16
@@ -106,6 +106,12 @@ def kernel_warmup(worker: "Worker"):
106
)
107
108
109
+# TODO: remove once FlashInfer upstream fixes the persistent file cache
110
+# to resolve collisions like `use_8x4_sf_layout=True/False`, which causes
111
+# invalid tactics to be chosen
112
+_FLASHINFER_USE_PERSISTENT_CACHE = False
113
+
114
+
115
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
116
"""
117
Autotune FlashInfer operations.
@@ -122,6 +128,16 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
128
import vllm.utils.flashinfer as fi_utils
129
from vllm.distributed.parallel_state import get_world_group
130
131
+ if not _FLASHINFER_USE_PERSISTENT_CACHE:
132
+ with torch.inference_mode(), fi_utils.autotune():
133
+ runner._dummy_run(
134
+ num_tokens=runner.scheduler_config.max_num_batched_tokens,
135
+ skip_eplb=True,
136
+ is_profile=True,
137
+ )
138
+ get_world_group().barrier()
139
+ return
140
+
141
world = get_world_group()
142
is_leader = world.rank_in_group == 0
143