Bump flashinfer version to 0.6.13 (#46683)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Wei Zhao committed
Jun 29, 2026 at 12:30 UTC
4708292d48f3f15978a6ad3befe5f0052bc86491
6 files changed
+29
-15
docker/Dockerfile
+1
-1
@@ -793,7 +793,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
793
# Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
794
# https://docs.flashinfer.ai/installation.html
795
# From versions.json: .flashinfer.version
796
-ARG FLASHINFER_VERSION=0.6.12
796
+ARG FLASHINFER_VERSION=0.6.13
797
RUN --mount=type=cache,target=/opt/uv/cache \
798
uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
799
--index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
docker/Dockerfile.nightly_torch
+2
-2
@@ -257,13 +257,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
257
258
259
# build flashinfer for torch nightly from source around 10 mins
260
-# release version: v0.6.12
260
+# release version: v0.6.13
261
# todo(elainewy): cache flashinfer build result for faster build
262
ENV CCACHE_DIR=/root/.cache/ccache
263
RUN --mount=type=cache,target=/root/.cache/ccache \
264
--mount=type=cache,target=/root/.cache/uv \
265
echo "git clone flashinfer..." \
266
- && git clone --depth 1 --branch v0.6.12 --recursive https://github.com/flashinfer-ai/flashinfer.git \
266
+ && git clone --depth 1 --branch v0.6.13 --recursive https://github.com/flashinfer-ai/flashinfer.git \
267
&& cd flashinfer \
268
&& git submodule update --init --recursive \
269
&& echo "finish git clone flashinfer..." \
docker/versions.json
+1
-1
@@ -68,7 +68,7 @@
68
"default": "true"
69
},
70
"FLASHINFER_VERSION": {
71
- "default": "0.6.12"
71
+ "default": "0.6.13"
72
},
73
"GDRCOPY_CUDA_VERSION": {
74
"default": "12.8"
requirements/cuda.txt
+2
-2
@@ -10,8 +10,8 @@ torchaudio==2.11.0
10
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
11
PyNvVideoCodec==2.0.4
12
# FlashInfer should be updated together with the Dockerfile
13
-flashinfer-python==0.6.12
14
-flashinfer-cubin==0.6.12
13
+flashinfer-python==0.6.13
14
+flashinfer-cubin==0.6.13
15
apache-tvm-ffi==0.1.9
16
tilelang==0.1.9
17
nvidia-cudnn-frontend>=1.19.1
tests/evals/gsm8k/test_gsm8k_correctness.py
+9
-2
@@ -33,6 +33,8 @@ QUARK_MXFP4_TORCH_COMPATIBLE = find_spec("quark") is not None and (
33
else True
34
)
35
36
+DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200
37
+
38
39
def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
40
"""Run GSM8K evaluation using our isolated script."""
@@ -127,7 +129,11 @@ def test_gsm8k_correctness(config_filename):
129
]
130
)
131
130
- env_dict = eval_config.get("env", None)
132
+ startup_max_wait_seconds = eval_config.get(
133
+ "startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS
134
+ )
135
+ env_dict = dict(eval_config.get("env") or {})
136
+ env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds))
137
138
print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
139
print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
@@ -139,6 +145,7 @@ def test_gsm8k_correctness(config_filename):
145
"rocm_request_timeout_seconds", request_timeout_seconds
146
)
147
print(f"Request timeout: {request_timeout_seconds}s")
148
+ print(f"Startup max wait: {startup_max_wait_seconds}s")
149
print(f"Server args: {' '.join(server_args)}")
150
print(f"Environment variables: {env_dict}")
151
@@ -147,7 +154,7 @@ def test_gsm8k_correctness(config_filename):
154
eval_config["model_name"],
155
server_args,
156
env_dict=env_dict,
150
- max_wait_seconds=eval_config.get("startup_max_wait_seconds", 600),
157
+ max_wait_seconds=startup_max_wait_seconds,
158
) as remote_server:
159
server_url = remote_server.url_for("v1")
160
print(f"Server started at: {server_url}")
vllm/model_executor/warmup/kernel_warmup.py
+14
-7
@@ -113,12 +113,6 @@ def kernel_warmup(worker: "Worker"):
113
)
114
115
116
-# TODO: remove once FlashInfer upstream fixes the persistent file cache
117
-# to resolve collisions like `use_8x4_sf_layout=True/False`, which causes
118
-# invalid tactics to be chosen
119
-_FLASHINFER_USE_PERSISTENT_CACHE = False
120
-
121
-
116
def flashinfer_autotune(runner: "GPUModelRunner") -> None:
117
"""
118
Autotune FlashInfer operations.
@@ -135,7 +129,20 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
129
import vllm.utils.flashinfer as fi_utils
130
from vllm.distributed.parallel_state import get_world_group
131
138
- if not _FLASHINFER_USE_PERSISTENT_CACHE:
132
+ use_persistent_cache = True
133
+
134
+ deepep_a2a_backends = {
135
+ "deepep_high_throughput",
136
+ "deepep_low_latency",
137
+ "deepep_v2",
138
+ }
139
+ if runner.vllm_config.parallel_config.all2all_backend in deepep_a2a_backends:
140
+ # DeepEP dispatch/combine can timeout when only rank 0
141
+ # performs autotune and falls behind other ranks.
142
+ # Thus we skip persistent cache in this case.
143
+ use_persistent_cache = False
144
+
145
+ if not use_persistent_cache:
146
with torch.inference_mode(), fi_utils.autotune():
147
runner._dummy_run(
148
num_tokens=runner.scheduler_config.max_num_batched_tokens,